Training in progress epoch 0

.gitattributes

@@ -35,3 +35,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 output/source.spm filter=lfs diff=lfs merge=lfs -text
 output/target.spm filter=lfs diff=lfs merge=lfs -text
+myoutput_1541.out filter=lfs diff=lfs merge=lfs -text

myerrors_1540.out

@@ -0,0 +1 @@
+
  0%|          | 0/329 [00:00<?, ?it/s]
  1%|          | 2/329 [00:01<04:21,  1.25it/s]
  1%|          | 3/329 [00:03<06:34,  1.21s/it]
  1%|          | 4/329 [00:05<08:35,  1.59s/it]
  2%|▏         | 5/329 [00:07<09:49,  1.82s/it]slurmstepd-dl: error: *** JOB 1540 ON dl CANCELLED AT 2023-09-17T15:41:03 ***

myerrors_1541.out

@@ -0,0 +1 @@
+
  0%|          | 0/329 [00:00<?, ?it/s]
  1%|          | 2/329 [00:01<04:21,  1.25it/s]
  1%|          | 3/329 [00:03<07:00,  1.29s/it]
  1%|          | 4/329 [00:05<08:53,  1.64s/it]
  2%|▏         | 5/329 [00:08<10:03,  1.86s/it]
  2%|▏         | 6/329 [00:10<10:50,  2.01s/it]
  2%|▏         | 7/329 [00:12<11:24,  2.12s/it]
  2%|▏         | 8/329 [00:14<10:09,  1.90s/it]
  3%|▎         | 9/329 [00:15<09:10,  1.72s/it]
  3%|▎         | 10/329 [00:17<10:08,  1.91s/it]slurmstepd-dl: error: *** JOB 1541 ON dl CANCELLED AT 2023-09-17T15:42:50 ***

myerrors_1542.out

@@ -0,0 +1 @@
+
  0%|          | 0/329 [00:00<?, ?it/s]
  1%|          | 2/329 [00:01<04:19,  1.26it/s]
  1%|          | 3/329 [00:03<06:32,  1.20s/it]
  1%|          | 4/329 [00:05<08:33,  1.58s/it]
  2%|▏         | 5/329 [00:07<09:47,  1.81s/it]
  2%|▏         | 6/329 [00:10<10:36,  1.97s/it]
  2%|▏         | 7/329 [00:12<11:11,  2.09s/it]
  2%|▏         | 8/329 [00:13<09:59,  1.87s/it]
  3%|▎         | 9/329 [00:15<09:03,  1.70s/it]
  3%|▎         | 10/329 [00:17<10:02,  1.89s/it]
  3%|▎         | 11/329 [00:19<10:23,  1.96s/it]
  4%|▎         | 12/329 [00:21<11:01,  2.09s/it]
  4%|▍         | 13/329 [00:23<10:42,  2.03s/it]
  4%|▍         | 14/329 [00:26<10:53,  2.07s/it]
  5%|▍         | 15/329 [00:27<09:38,  1.84s/it]
  5%|▍         | 16/329 [00:28<08:35,  1.65s/it]
  5%|▌         | 17/329 [00:30<09:46,  1.88s/it]
  5%|▌         | 18/329 [00:33<10:21,  2.00s/it]
  6%|▌         | 19/329 [00:35<10:42,  2.07s/it]
  6%|▌         | 20/329 [00:37<10:59,  2.14s/it]
  6%|▋         | 21/329 [00:40<11:21,  2.21s/it]
  7%|▋         | 22/329 [00:42<10:52,  2.13s/it]
  7%|▋         | 23/329 [00:44<10:59,  2.16s/it]
  7%|▋         | 24/329 [00:46<11:03,  2.17s/it]
  8%|▊         | 25/329 [00:48<10:41,  2.11s/it]
  8%|▊         | 26/329 [00:50<11:01,  2.18s/it]
  8%|▊         | 27/329 [00:53<11:05,  2.20s/it]
  9%|▊         | 28/329 [00:55<11:17,  2.25s/it]
  9%|▉         | 29/329 [00:57<11:22,  2.27s/it]
  9%|▉         | 30/329 [00:59<09:55,  1.99s/it]
  9%|▉         | 31/329 [01:00<09:18,  1.87s/it]
 10%|▉         | 32/329 [01:02<09:51,  1.99s/it]
 10%|█         | 33/329 [01:05<10:15,  2.08s/it]
 10%|█         | 34/329 [01:07<10:15,  2.09s/it]
 11%|█         | 35/329 [01:09<10:28,  2.14s/it]
 11%|█         | 36/329 [01:11<10:32,  2.16s/it]
 11%|█         | 37/329 [01:13<09:45,  2.00s/it]
 12%|█▏        | 38/329 [01:15<09:46,  2.02s/it]
 12%|█▏        | 39/329 [01:16<08:37,  1.78s/it]
 12%|█▏        | 40/329 [01:18<07:56,  1.65s/it]
 12%|█▏        | 41/329 [01:20<08:40,  1.81s/it]
 13%|█▎        | 42/329 [01:22<09:24,  1.97s/it]
 13%|█▎        | 43/329 [01:24<09:45,  2.05s/it]
 13%|█▎        | 44/329 [01:27<10:12,  2.15s/it]
 14%|█▎        | 45/329 [01:29<10:28,  2.21s/it]
 14%|█▍        | 46/329 [01:31<10:22,  2.20s/it]
 14%|█▍        | 47/329 [01:33<10:08,  2.16s/it]
 15%|█▍        | 48/329 [01:35<09:55,  2.12s/it]
 15%|█▍        | 49/329 [01:38<10:02,  2.15s/it]
 15%|█▌        | 50/329 [01:40<10:07,  2.18s/it]
 16%|█▌        | 51/329 [01:42<10:11,  2.20s/it]
 16%|█▌        | 52/329 [01:44<10:21,  2.25s/it]
 16%|█▌        | 53/329 [01:47<10:13,  2.22s/it]
 16%|█▋        | 54/329 [01:48<08:43,  1.90s/it]
 17%|█▋        | 55/329 [01:50<09:10,  2.01s/it]
 17%|█▋        | 56/329 [01:52<09:07,  2.00s/it]
 17%|█▋        | 57/329 [01:54<09:21,  2.06s/it]
 18%|█▊        | 58/329 [01:56<09:14,  2.05s/it]
 18%|█▊        | 59/329 [01:58<09:29,  2.11s/it]
 18%|█▊        | 60/329 [02:01<09:48,  2.19s/it]
 19%|█▊        | 61/329 [02:03<10:00,  2.24s/it]
 19%|█▉        | 62/329 [02:05<09:48,  2.20s/it]
 19%|█▉        | 63/329 [02:08<09:54,  2.24s/it]
 19%|█▉        | 64/329 [02:10<09:30,  2.15s/it]
 20%|█▉        | 65/329 [02:12<09:37,  2.19s/it]
 20%|██        | 66/329 [02:14<09:36,  2.19s/it]
 20%|██        | 67/329 [02:16<09:20,  2.14s/it]
 21%|██        | 68/329 [02:18<09:28,  2.18s/it]
 21%|██        | 69/329 [02:21<09:39,  2.23s/it]
 21%|██▏       | 70/329 [02:23<09:42,  2.25s/it]
 22%|██▏       | 71/329 [02:25<09:41,  2.25s/it]
 22%|██▏       | 72/329 [02:28<09:41,  2.26s/it]
 22%|██▏       | 73/329 [02:29<09:15,  2.17s/it]
 22%|██▏       | 74/329 [02:32<09:30,  2.24s/it]
 23%|██▎       | 75/329 [02:34<09:36,  2.27s/it]
 23%|██▎       | 76/329 [02:36<09:30,  2.26s/it]
 23%|██▎       | 77/329 [02:39<09:31,  2.27s/it]
 24%|██▎       | 78/329 [02:40<08:44,  2.09s/it]
 24%|██▍       | 79/329 [02:42<08:37,  2.07s/it]
 24%|██▍       | 80/329 [02:45<08:50,  2.13s/it]
 25%|██▍       | 81/329 [02:47<09:13,  2.23s/it]
 25%|██▍       | 82/329 [02:49<09:09,  2.23s/it]
 25%|██▌       | 83/329 [02:52<09:15,  2.26s/it]
 26%|██▌       | 84/329 [02:54<08:40,  2.12s/it]
 26%|██▌       | 85/329 [02:56<08:44,  2.15s/it]
 26%|██▌       | 86/329 [02:57<07:48,  1.93s/it]
 26%|██▋       | 87/329 [02:59<07:16,  1.81s/it]
 27%|██▋       | 88/329 [03:01<07:41,  1.92s/it]
 27%|██▋       | 89/329 [03:03<08:04,  2.02s/it]
 27%|██▋       | 90/329 [03:05<08:22,  2.10s/it]
 28%|██▊       | 91/329 [03:08<08:28,  2.14s/it]
 28%|██▊       | 92/329 [03:10<08:20,  2.11s/it]
 28%|██▊       | 93/329 [03:12<08:41,  2.21s/it]
 29%|██▊       | 94/329 [03:13<07:39,  1.96s/it]
 29%|██▉       | 95/329 [03:16<07:52,  2.02s/it]
 29%|██▉       | 96/329 [03:18<07:47,  2.01s/it]
 29%|██▉       | 97/329 [03:20<08:07,  2.10s/it]
 30%|���█▉       | 98/329 [03:22<08:13,  2.14s/it]
 30%|███       | 99/329 [03:24<08:22,  2.18s/it]
 30%|███       | 100/329 [03:27<08:29,  2.22s/it]
 31%|███       | 101/329 [03:28<07:34,  1.99s/it]
 31%|███       | 102/329 [03:30<06:44,  1.78s/it]
 31%|███▏      | 103/329 [03:32<07:09,  1.90s/it]
 32%|███▏      | 104/329 [03:34<07:39,  2.04s/it]
 32%|███▏      | 105/329 [03:36<07:00,  1.88s/it]
 32%|███▏      | 106/329 [03:38<07:17,  1.96s/it]
 33%|███▎      | 107/329 [03:40<07:35,  2.05s/it]
 33%|███▎      | 108/329 [03:42<07:03,  1.92s/it]
 33%|███▎      | 109/329 [03:44<07:14,  1.98s/it]
 33%|███▎      | 110/329 [03:45<06:54,  1.89s/it]
 34%|███▎      | 111/329 [03:47<06:02,  1.66s/it]
 34%|███▍      | 112/329 [03:49<06:36,  1.83s/it]
 34%|███▍      | 113/329 [03:51<06:47,  1.89s/it]
 35%|███▍      | 114/329 [03:52<06:13,  1.74s/it]
 35%|███▍      | 115/329 [03:54<06:42,  1.88s/it]
 35%|███▌      | 116/329 [03:57<07:06,  2.00s/it]
 36%|███▌      | 117/329 [03:59<07:06,  2.01s/it]
 36%|███▌      | 118/329 [04:00<06:31,  1.85s/it]
 36%|███▌      | 119/329 [04:02<06:19,  1.81s/it]
 36%|███▋      | 120/329 [04:04<06:46,  1.94s/it]
 37%|███▋      | 121/329 [04:06<06:28,  1.87s/it]
 37%|███▋      | 122/329 [04:08<06:38,  1.92s/it]
 37%|███▋      | 123/329 [04:10<06:53,  2.01s/it]
 38%|███▊      | 124/329 [04:12<07:01,  2.06s/it]
 38%|███▊      | 125/329 [04:15<07:12,  2.12s/it]
 38%|███▊      | 126/329 [04:17<07:15,  2.14s/it]
 39%|███▊      | 127/329 [04:19<07:26,  2.21s/it]
 39%|███▉      | 128/329 [04:21<07:04,  2.11s/it]
 39%|███▉      | 129/329 [04:23<07:03,  2.12s/it]slurmstepd-dl: error: *** JOB 1542 ON dl CANCELLED AT 2023-09-17T15:47:56 ***

myoutput_1540.out

@@ -0,0 +1,27 @@
+DatasetDict({
+    train: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 210173
+    })
+})
+DatasetDict({
+    train: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 189155
+    })
+    test: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 21018
+    })
+})
+DatasetDict({
+    train: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 189155
+    })
+    validation: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 21018
+    })
+})
+evaluate1

myoutput_1541.out

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3954e1269fd8ff6494d0f93177fa00f2346a9995299621d2cb927da763b95625
+size 23953172

myoutput_1542.out

@@ -0,0 +1,12 @@
+[{'en': "Calibration is about to check the value range your device delivers. Please move axis %1 %2 on your device to the maximum position. Press any button on the device or click on the'Next 'button to continue with the next step.", 'fr': "Le calibrage va vérifier la plage de valeurs que votre matériel produit. Veuillez déplacer l'axe %1 %2 de votre périphérique à la position maximale. Appuyez sur n'importe quel bouton du périphérique ou sur le bouton « & #160; Suivant & #160; » pour la prochaine étape."}, {'en': 'Default to expanded threads', 'fr': 'Par défaut, développer les fils de discussion'}, {'en': '*. ui *. UI_BAR_User Interface Files', 'fr': '*. ui *. UI_BAR_Fichiers interface utilisateur'}, {'en': 'New Action', 'fr': 'Nouvelle action'}, {'en': 'Sentence, word and letter counts for this document', 'fr': 'Nombre de phrases, de mots et de lettres pour ce document'}, {'en': 'Reorder Thumbnails', 'fr': 'Réordonner les vignettes'}, {'en': "Anubis, or Inpu, was an Egyptian god of the afterlife. To get the treasure between Anubis' ears, dig out the two columns to the left of the mummy and stand on top of the mummy. Move left off the mummy while digging right. The mummy should fall into the hole in the right most column. Now dig out the left two columns until you connect to the ladder-eye. Move down from the right most ladder of the eye and you will fall to the pole below. To get the treasure inside the right ear, dig out the higher of the two-block columns, falling onto the right two-block column. Move to the left and dig the second two-block column. Now move to the right and dig out the one-block column, freeing the mummy. Move one more to the right and dig out three blocks so that you have a two block wide ledge to stand on. Move left and dig out the square to your right. Wait for the mummy to fall, get his treasure, and race up the ladder to confront five mummies blocking the exit.", 'fr': "Anubis, ou Inpou & #160; / Anepou, était un dieu égyptien de la vie de l'au-delà. Pour récupérer le trésor entre les oreilles d'Anubis, creusez les deux colonnes à la gauche de la momie et marchez sur le haut de la momie. Déplacez vous à gauche de la momie en creusant vers la droite. La momie devrait tomber dans le trou dans la colonne la plus à droite. Maintenant creusez les deux colonnes de gauche jusqu'à ce que vous vous connectez à l 'œ il-échelle. Descendez de l'échelle la plus à droite de l 'œ il et vous tomberez dans la fosse en dessous. Pour récupérer le trésor dans l'oreille de droite, creusez au plus haut des deux blocs de colonnes, en tombant sur la droite de celles -ci. Déplacez vous vers la gauche et creusez le deuxième deux-blocs de colonnes, libérant la momie. Déplacez vous une fois de plus vers la droite et creusez les trois blocs pour avoir un large rebord de deux blocs pour marcher. Déplacez vous vers la gauche et creusez la carré à votre droite. Attendez que la momie tombe, récupérez le trésor, et courrez sur l'échelle pour faire face au cinq momies bloquant la sortie."}, {'en': 'Quickly get the first two enemies into the double pit to the left of the gold. First get both of them into the little box, then release them left. Now fall onto them and collect the gold, then ride the third enemy down and use him to get across to the ladder.', 'fr': "Mettez rapidement les deux premiers ennemis dans la double fosse à gauche de l'or. Tout d'abord, mettez -les tous deux dans la sorte de petite boîte, puis relâchez -les à gauche. Maintenant, laissez -vous tomber sur eux et collectez l'or, puis faites tomber le troisième ennemi et utilisez -le pour parvenir à l'échelle."}, {'en': '+50', 'fr': '+50'}, {'en': 'YAML', 'fr': 'YAMLLanguage'}]
+DatasetDict({
+    train: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 189155
+    })
+    validation: Dataset({
+        features: ['id', 'translation'],
+        num_rows: 21018
+    })
+})
+evaluate1

translation.py

@@ -22,9 +22,11 @@ from torch import Tensor
 model_checkpoint = "Helsinki-NLP/opus-mt-en-fr"
 tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, return_tensors="pt")
 raw_datasets = load_dataset("kde4", lang1="en", lang2="fr")
+
 split_datasets = raw_datasets["train"].train_test_split(train_size=0.9, seed=20)
+print(split_datasets["train"]["translation"][0:10])
 split_datasets["validation"] = split_datasets.pop("test")
-
+print(split_datasets)
 #contants
 max_length = 128