File size: 27,365 Bytes
d028f33
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
Namespace(data_path='/scratch/eo41/data/saycam/A_5fps_300s_{000000..000002}.tar', vqconfig_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.yaml', vqmodel_path='/scratch/eo41/vqgan-gpt/vqgan_pretrained_models/a_32x32_8192.ckpt', num_workers=16, seed=0, save_dir='/scratch/eo41/vqgan-gpt/gpt_pretrained_models', save_prefix='a_gimel', gpt_config='GPT_gimel', vocab_size=8192, block_size=1023, batch_size=6, print_freq=4000, lr=0.0003, optimizer='Adam', resume='/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt', gpu=None, world_size=-1, rank=-1, dist_url='env://', dist_backend='nccl', local_rank=-1)
model:
  base_learning_rate: 1.0e-05
  params:
    ddconfig:
      attn_resolutions:
      - 32
      ch: 128
      ch_mult:
      - 1
      - 1
      - 2
      - 4
      double_z: false
      dropout: 0.0
      in_channels: 3
      num_res_blocks: 2
      out_ch: 3
      resolution: 256
      z_channels: 256
    embed_dim: 256
    lossconfig:
      params:
        codebook_weight: 1.0
        disc_conditional: false
        disc_in_channels: 3
        disc_start: 100001
        disc_weight: 0.2
      target: vqloss.VQLPIPSWithDiscriminator
    n_embed: 8192
  target: vqmodel.VQModel

Working with z of shape (1, 256, 32, 32) = 262144 dimensions.
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
  warnings.warn(
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
  warnings.warn(msg)
loaded pretrained LPIPS loss from taming/modules/autoencoder/lpips/vgg.pth
VQLPIPSWithDiscriminator running with hinge loss.
Number of parameters: 730671360
Running on 16 GPUs total
=> loaded model weights and optimizer state at checkpoint '/scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel.pt'
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
/scratch/eo41/miniconda3/lib/python3.10/site-packages/torch/nn/_reduction.py:42: UserWarning: size_average and reduce args will be deprecated, please use reduction='none' instead.
  warnings.warn(warning.format(ret))
Iteration: 0 | Training loss: 4.586821556091309
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_0.pt
Iteration: 4000 | Training loss: 5.206405858278274
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_4000.pt
Iteration: 8000 | Training loss: 5.007484036445618
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_8000.pt
Iteration: 12000 | Training loss: 4.913228427648544
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_12000.pt
Iteration: 16000 | Training loss: 4.878944065392018
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_16000.pt
Iteration: 20000 | Training loss: 4.853666521191597
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_20000.pt
Iteration: 24000 | Training loss: 4.85290568870306
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_24000.pt
Iteration: 28000 | Training loss: 4.853768558263779
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_28000.pt
Iteration: 32000 | Training loss: 4.846220250189305
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_32000.pt
Iteration: 36000 | Training loss: 4.852501285433769
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_36000.pt
Iteration: 40000 | Training loss: 4.8531693903803825
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_40000.pt
Iteration: 44000 | Training loss: 4.841027199268341
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_44000.pt
Iteration: 48000 | Training loss: 4.852847850620747
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_48000.pt
Iteration: 52000 | Training loss: 4.848223978936672
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_52000.pt
Iteration: 56000 | Training loss: 4.845628525495529
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_56000.pt
Iteration: 60000 | Training loss: 4.834981297135353
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_60000.pt
Iteration: 64000 | Training loss: 4.828589959859848
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_64000.pt
Iteration: 68000 | Training loss: 4.826472416520119
Saving model to: /scratch/eo41/vqgan-gpt/gpt_pretrained_models/a_gimel_68000.pt
slurmstepd: error: *** JOB 29938000 ON ga001 CANCELLED AT 2023-02-09T22:22:39 ***
srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
slurmstepd: error: *** STEP 29938000.0 ON ga001 CANCELLED AT 2023-02-09T22:22:39 ***