diff --git "a/logs/main/main_log.txt" "b/logs/main/main_log.txt" new file mode 100644--- /dev/null +++ "b/logs/main/main_log.txt" @@ -0,0 +1,10001 @@ +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +WARNING:__main__: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]: warnings.warn( +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default7]:> setting tensorboard ... +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:using world size: 64, data-parallel-size: 32, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 +[default0]:WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:PretrainedFromHF +[default0]:using torch.float16 for parameters ... +[default0]:------------------------ arguments ------------------------ +[default0]: abort_on_unmet_fused_kernel_constraints ......... True +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]: accumulate_allreduce_grads_in_fp32 .............. False +[default0]: adam_beta1 ...................................... 0.9 +[default0]: adam_beta2 ...................................... 0.95 +[default0]: adam_eps ........................................ 1e-08 +[default0]: adlr_autoresume ................................. False +[default0]: adlr_autoresume_interval ........................ 1000 +[default0]: apply_query_key_layer_scaling ................... True +[default0]: apply_residual_connection_post_layernorm ........ False +[default0]: attention_dropout ............................... 0.1 +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default0]: attention_softmax_in_fp32 ....................... False +[default0]: bert_binary_head ................................ True +[default0]: bert_load ....................................... None +[default0]: bf16 ............................................ False +[default0]: bias_dropout_fusion ............................. True +[default0]: bias_gelu_fusion ................................ True +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]: biencoder_projection_dim ........................ 0 +[default0]: biencoder_shared_query_context_model ............ False +[default0]: block_data_path ................................. None +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default0]: checkpoint_activations .......................... True +[default0]: checkpoint_in_cpu ............................... False +[default0]: checkpoint_num_layers ........................... 1 +[default0]: clip_grad ....................................... 1.0 +[default0]: codecarbon_dir .................................. None +[default0]: consumed_train_samples .......................... 0 +[default0]: consumed_train_tokens ........................... 0 +[default0]: consumed_valid_samples .......................... 0 +[default0]: contigious_checkpointing ........................ False +[default0]: cpu_optimizer ................................... False +[default0]: cpu_torch_adam .................................. False +[default0]: curriculum_learning ............................. False +[default0]: data_impl ....................................... mmap +[default0]: data_parallel_size .............................. 32 +[default0]: data_path ....................................... ['/gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document'] +[default0]: dataloader_type ................................. single +[default0]: DDP_impl ........................................ local +[default0]: decoder_seq_length .............................. None +[default0]: deepscale ....................................... False +[default0]: deepscale_config ................................ None +[default0]: deepspeed ....................................... True +[default0]: deepspeed_activation_checkpointing .............. True +[default0]: deepspeed_config ................................ ./ds_config.1219018.json +[default0]: deepspeed_mpi ................................... False +[default0]: distribute_checkpointed_activations ............. False +[default0]: distributed_backend ............................. nccl +[default0]: embed_layernorm ................................. True +[default0]: embedding_path .................................. None +[default0]: encoder_seq_length .............................. 2048 +[default0]: eod_mask_loss ................................... False +[default0]: eval_interval ................................... 1000 +[default0]: eval_iters ...................................... 1 +[default0]: eval_only ....................................... None +[default0]: evidence_data_path .............................. None +[default0]: exit_duration_in_mins ........................... 1190 +[default0]: exit_interval ................................... None +[default0]: ffn_hidden_size ................................. 10240 +[default0]: finetune ........................................ False +[default0]: fp16 ............................................ True +[default0]: fp16_lm_cross_entropy ........................... False +[default0]: fp32_residual_connection ........................ False +[default0]: gigaflos_no_embeds .............................. 0 +[default0]: global_batch_size ............................... 512 +[default0]: glu_activation .................................. None +[default0]: hidden_dropout .................................. 0.1 +[default0]: hidden_size ..................................... 2560 +[default0]: hysteresis ...................................... 2 +[default0]: ict_head_size ................................... None +[default0]: ict_load ........................................ None +[default0]: img_dim ......................................... 224 +[default0]: indexer_batch_size .............................. 128 +[default0]: indexer_log_interval ............................ 1000 +[default0]: inference ....................................... False +[default0]: init_method_std ................................. 0.1 +[default0]: init_method_xavier_uniform ...................... False +[default0]: initial_loss_scale .............................. 4294967296 +[default0]: kill_switch_path ................................ None +[default0]: kv_channels ..................................... 80 +[default0]: layernorm_epsilon ............................... 1e-05 +[default0]: lazy_mpu_init ................................... None +[default0]: load ............................................ /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]: local_rank ...................................... None +[default0]: log_batch_size_to_tensorboard ................... True +[default0]: log_interval .................................... 1 +[default0]: log_learning_rate_to_tensorboard ................ True +[default0]: log_level ....................................... None +[default0]: log_level_replica ............................... None +[default0]: log_loss_scale_to_tensorboard ................... True +[default0]: log_num_zeros_in_grad ........................... False +[default0]: log_params_norm ................................. False +[default0]: log_path ........................................ None +[default0]: log_timers_to_tensorboard ....................... True +[default0]: log_validation_ppl_to_tensorboard ............... True +[default0]: loss_on_targets_only ............................ False +[default0]: loss_scale ...................................... None +[default0]: loss_scale_window ............................... 1000 +[default0]: lr .............................................. 0.01 +[default0]: lr_decay_iters .................................. None +[default0]: lr_decay_samples ................................ 1953125 +[default0]: lr_decay_style .................................. cosine +[default0]: lr_decay_tokens ................................. None +[default0]: lr_warmup_fraction .............................. None +[default0]: lr_warmup_iters ................................. 0 +[default0]: lr_warmup_samples ............................... 183105 +[default0]: make_vocab_size_divisible_by .................... 128 +[default0]: mask_prob ....................................... 0.15 +[default0]: masked_softmax_fusion ........................... True +[default0]: max_position_embeddings ......................... 2048 +[default0]: mean_noise_span_length .......................... None +[default0]: memory_centric_tiled_linear ..................... False +[default0]: merge_file ...................................... None +[default0]: micro_batch_size ................................ 16 +[default0]: min_loss_scale .................................. 1.0 +[default0]: min_lr .......................................... 0.001 +[default0]: mmap_warmup ..................................... False +[default0]: mup ............................................. True +[default0]: mup_attn_mult ................................... 10.0 +[default0]: mup_base_ffn_hidden_size ........................ 256 +[default0]: mup_base_hidden_size ............................ 64 +[default0]: mup_coord_check ................................. False +[default0]: mup_input_mult .................................. 10.0 +[default0]: mup_output_mult ................................. 10.0 +[default0]: no_load_optim ................................... None +[default0]: no_load_rng ..................................... None +[default0]: no_save_optim ................................... None +[default0]: no_save_rng ..................................... None +[default0]: noise_density ................................... None +[default0]: num_attention_heads ............................. 32 +[default0]: num_channels .................................... 3 +[default0]: num_classes ..................................... 1000 +[default0]: num_layers ...................................... 32 +[default0]: num_layers_per_virtual_pipeline_stage ........... None +[default0]: num_workers ..................................... 2 +[default0]: onnx_safe ....................................... None +[default0]: openai_gelu ..................................... False +[default0]: optimizer ....................................... adam +[default0]: override_lr_scheduler ........................... False +[default0]: pad_vocab_size_to ............................... 51200 +[default0]: params_dtype .................................... torch.float16 +[default0]: partition_activations ........................... False +[default0]: patch_dim ....................................... 16 +[default0]: pipeline_model_parallel_size .................... 1 +[default0]: position_embedding_type ......................... PositionEmbeddingType.alibi +[default0]: pp_partition_method ............................. type:transformer +[default0]: profile_backward ................................ False +[default0]: query_in_block_prob ............................. 0.1 +[default0]: rampup_batch_size ............................... None +[default0]: rank ............................................ 0 +[default0]: remote_device ................................... none +[default0]: reset_attention_mask ............................ False +[default0]: reset_position_ids .............................. False +[default0]: retriever_report_topk_accuracies ................ [] +[default0]: retriever_score_scaling ......................... False +[default0]: retriever_seq_length ............................ 256 +[default0]: reweight_loss_based_on_position_frequency ....... False +[default0]: sample_rate ..................................... 1.0 +[default0]: save ............................................ /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]: save_interval ................................... 250 +[default0]: scatter_gather_tensors_in_pipeline .............. True +[default0]: scattered_embeddings ............................ False +[default0]: seed ............................................ 42 +[default0]: seq_length ...................................... 2048 +[default0]: sgd_momentum .................................... 0.9 +[default0]: short_seq_prob .................................. 0.1 +[default0]: skip_train_iteration_range ...................... None +[default0]: split ........................................... 969, 30, 1 +[default0]: split_transformers .............................. False +[default0]: sync_tp_duplicated_parameters ................... False +[default0]: synchronize_each_layer .......................... False +[default0]: tensor_model_parallel_size ...................... 2 +[default0]: tensorboard_dir ................................. /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/tr14-2B7-test-lr0.01-init0.1-inpm10-outm10-atnm10-mup/tensorboard/main +[default0]: tensorboard_log_interval ........................ 1 +[default0]: tensorboard_queue_size .......................... 5 +[default0]: test_weighted_split_names ....................... None +[default0]: test_weighted_split_paths ....................... None +[default0]: test_weighted_split_paths_path .................. None +[default0]: test_weighted_split_splits ...................... None +[default0]: test_weighted_split_weights ..................... None +[default0]: tile_factor ..................................... 1 +[default0]: titles_data_path ................................ None +[default0]: tokenizer_name_or_path .......................... t5-small +[default0]: tokenizer_type .................................. PretrainedFromHF +[default0]: train_iters ..................................... None +[default0]: train_samples ................................... 1953125 +[default0]: train_tokens .................................... None +[default0]: train_weighted_split_paths ...................... None +[default0]: train_weighted_split_paths_path ................. None +[default0]: universal_checkpoint ............................ False +[default0]: use_bnb_optimizer ............................... False +[default0]: use_checkpoint_lr_scheduler ..................... False +[default0]: use_contiguous_buffers_in_ddp ................... False +[default0]: use_cpu_initialization .......................... None +[default0]: use_one_sent_docs ............................... False +[default0]: use_pin_memory .................................. False +[default0]: valid_num_workers ............................... 2 +[default0]: valid_weighted_split_names ...................... None +[default0]: valid_weighted_split_paths ...................... None +[default0]: valid_weighted_split_paths_path ................. None +[default0]: valid_weighted_split_splits ..................... None +[default0]: valid_weighted_split_weights .................... None +[default0]: virtual_pipeline_model_parallel_size ............ None +[default0]: vocab_extra_ids ................................. 0 +[default0]: vocab_file ...................................... None +[default0]: weight_decay .................................... 0.1 +[default0]: world_size ...................................... 64 +[default0]: zero_allgather_bucket_size ...................... 0.0 +[default0]: zero_contigious_gradients ....................... False +[default0]: zero_reduce_bucket_size ......................... 0.0 +[default0]: zero_reduce_scatter ............................. False +[default0]: zero_stage ...................................... 1 +[default0]:-------------------- end of arguments --------------------- +[default0]:setting number of micro-batches to constant 1 +[default0]:> building PretrainedFromHF tokenizer ... +[default0]: vocab file is un-used. loading tokenizer from pre-trained model +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default0]:Offline mode: forcing local_files_only=True +[default0]:Could not locate the tokenizer configuration file, will try to use the model config instead. +[default0]:Offline mode: forcing local_files_only=True +[default0]:loading configuration file config.json from cache at /gpfswork/rech/ajs/commun/models/models--t5-small/snapshots/3479082dc36f8a4730936ef1c9b88cd8b0835c53/config.json +[default0]:Model config T5Config { +[default0]: "_name_or_path": "t5-small", +[default0]: "architectures": [ +[default0]: "T5ForConditionalGeneration" +[default0]: ], +[default0]: "d_ff": 2048, +[default0]: "d_kv": 64, +[default0]: "d_model": 512, +[default0]: "decoder_start_token_id": 0, +[default0]: "dense_act_fn": "relu", +[default0]: "dropout_rate": 0.1, +[default0]: "eos_token_id": 1, +[default0]: "feed_forward_proj": "relu", +[default0]: "initializer_factor": 1.0, +[default0]: "is_encoder_decoder": true, +[default0]: "is_gated_act": false, +[default0]: "layer_norm_epsilon": 1e-06, +[default0]: "model_type": "t5", +[default0]: "n_positions": 512, +[default0]: "num_decoder_layers": 6, +[default0]: "num_heads": 8, +[default0]: "num_layers": 6, +[default0]: "output_past": true, +[default0]: "pad_token_id": 0, +[default0]: "relative_attention_max_distance": 128, +[default0]: "relative_attention_num_buckets": 32, +[default0]: "task_specific_params": { +[default0]: "summarization": { +[default0]: "early_stopping": true, +[default0]: "length_penalty": 2.0, +[default0]: "max_length": 200, +[default0]: "min_length": 30, +[default0]: "no_repeat_ngram_size": 3, +[default0]: "num_beams": 4, +[default0]: "prefix": "summarize: " +[default0]: }, +[default0]: "translation_en_to_de": { +[default0]: "early_stopping": true, +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to German: " +[default0]: }, +[default0]: "translation_en_to_fr": { +[default0]: "early_stopping": true, +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to French: " +[default0]: }, +[default0]: "translation_en_to_ro": { +[default0]: "early_stopping": true, +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to Romanian: " +[default0]: } +[default0]: }, +[default0]: "transformers_version": "4.25.0.dev0", +[default0]: "use_cache": true, +[default0]: "vocab_size": 32128 +[default0]:} +[default0]: +[default0]:Offline mode: forcing local_files_only=True +[default0]:loading file spiece.model from cache at /gpfswork/rech/ajs/commun/models/models--t5-small/snapshots/3479082dc36f8a4730936ef1c9b88cd8b0835c53/spiece.model +[default0]:loading file tokenizer.json from cache at /gpfswork/rech/ajs/commun/models/models--t5-small/snapshots/3479082dc36f8a4730936ef1c9b88cd8b0835c53/tokenizer.json +[default0]:loading file added_tokens.json from cache at None +[default0]:loading file special_tokens_map.json from cache at None +[default0]:loading file tokenizer_config.json from cache at None +[default0]:loading configuration file config.json from cache at /gpfswork/rech/ajs/commun/models/models--t5-small/snapshots/3479082dc36f8a4730936ef1c9b88cd8b0835c53/config.json +[default0]:Model config T5Config { +[default0]: "_name_or_path": "t5-small", +[default0]: "architectures": [ +[default0]: "T5ForConditionalGeneration" +[default0]: ], +[default0]: "d_ff": 2048, +[default0]: "d_kv": 64, +[default0]: "d_model": 512, +[default0]: "decoder_start_token_id": 0, +[default0]: "dense_act_fn": "relu", +[default0]: "dropout_rate": 0.1, +[default0]: "eos_token_id": 1, +[default0]: "feed_forward_proj": "relu", +[default0]: "initializer_factor": 1.0, +[default0]: "is_encoder_decoder": true, +[default0]: "is_gated_act": false, +[default0]: "layer_norm_epsilon": 1e-06, +[default0]: "model_type": "t5", +[default0]: "n_positions": 512, +[default0]: "num_decoder_layers": 6, +[default0]: "num_heads": 8, +[default0]: "num_layers": 6, +[default0]: "output_past": true, +[default0]: "pad_token_id": 0, +[default0]: "relative_attention_max_distance": 128, +[default0]: "relative_attention_num_buckets": 32, +[default0]: "task_specific_params": { +[default0]: "summarization": { +[default0]: "early_stopping": true, +[default0]: "length_penalty": 2.0, +[default0]: "max_length": 200, +[default0]: "min_length": 30, +[default0]: "no_repeat_ngram_size": 3, +[default0]: "num_beams": 4, +[default0]: "prefix": "summarize: " +[default0]: }, +[default0]: "translation_en_to_de": { +[default0]: "early_stopping": true, +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to German: " +[default0]: }, +[default0]: "translation_en_to_fr": { +[default0]: "early_stopping": true, +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to French: " +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: }, +[default0]: "translation_en_to_ro": { +[default0]: "early_stopping": true, +[default0]: "max_length": 300, +[default0]: "num_beams": 4, +[default0]: "prefix": "translate English to Romanian: " +[default0]: } +[default0]: }, +[default0]: "transformers_version": "4.25.0.dev0", +[default0]: "use_cache": true, +[default2]: warnings.warn( +[default0]: "vocab_size": 32128 +[default0]:} +[default0]: +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default0]: > padded vocab (size: 32100) with 19100 dummy tokens (new size: 51200) +[default0]:DeepSpeed general environment info: +[default0]:torch install path ............... ['/gpfswork/rech/six/commun/conda/tr11-176B-ml/lib/python3.8/site-packages/torch'] +[default0]:torch version .................... 1.11.0+cu115 +[default0]:torch cuda version ............... 11.5 +[default0]:torch hip version ................ None +[default0]:nvcc version ..................... 11.4 +[default0]:deepspeed install path ........... ['/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/DeepSpeed/deepspeed'] +[default0]:deepspeed info ................... 0.7.2+2a644488, 2a644488, master +[default0]:deepspeed wheel compiled w. ...... torch 1.11, cuda 11.5 +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default6]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default7]: warnings.warn( +[default1]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default3]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default1]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default1]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default3]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default3]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default1]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default1]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default1]: warnings.warn( +[default3]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default3]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default3]: warnings.warn( +[default0]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default0]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default0]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default0]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default0]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default0]: warnings.warn( +[default6]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default6]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default6]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default6]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default6]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default6]: warnings.warn( +[default5]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default5]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default5]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default5]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default5]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default5]: warnings.warn( +[default7]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default2]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default7]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default7]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:/gpfsssd/worksf/projects/rech/six/commun/code/tr11-176B-ml/transformers/src/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5. +[default4]:For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`. +[default4]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]:- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding. +[default7]: warnings.warn( +[default2]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default2]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default4]:- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding. +[default4]:- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value. +[default2]: warnings.warn( +[default4]: warnings.warn( +[default0]:**** Git info for Megatron: git_hash=ef0c6b8 git_branch=mup_integration **** +[default0]:> initializing torch distributed ... +[default0]:[2023-02-16 06:19:48,356] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +[default0]:> initializing tensor model parallel with size 2 +[default0]:> initializing pipeline model parallel with size 1 +[default0]:> setting random seeds to 42 ... +[default0]:[2023-02-16 06:19:51,295] [INFO] [checkpointing.py:226:model_parallel_cuda_manual_seed] > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 2760 and data parallel seed: 42 +[default0]:> compiling dataset index builder ... +[default0]:make: Entering directory '/gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/data' +[default0]:make: Nothing to be done for 'default'. +[default0]:make: Leaving directory '/gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/data' +[default0]:>>> done with dataset index builder. Compilation time: 0.101 seconds +[default0]:> compiling and loading fused kernels ... +[default0]:Detected CUDA files, patching ldflags +[default0]:Emitting ninja build file /gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +[default0]:Building extension module scaled_upper_triang_masked_softmax_cuda... +[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +[default0]:ninja: no work to do. +[default0]:Loading extension module scaled_upper_triang_masked_softmax_cuda... +[default0]:Detected CUDA files, patching ldflags +[default0]:Emitting ninja build file /gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +[default0]:Building extension module scaled_masked_softmax_cuda... +[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +[default0]:ninja: no work to do. +[default0]:Loading extension module scaled_masked_softmax_cuda... +[default0]:Detected CUDA files, patching ldflags +[default0]:Emitting ninja build file /gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... +[default0]:Building extension module fused_mix_prec_layer_norm_cuda... +[default0]:Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) +[default0]:ninja: no work to do. +[default0]:Loading extension module fused_mix_prec_layer_norm_cuda... +[default0]:>>> done with compiling and loading fused kernels. Compilation time: 10.661 seconds +[default0]:time to initialize megatron (seconds): 2.058 +[default0]:[after megatron is initialized] datetime: 2023-02-16 06:20:02 +[default0]:building GPT model ... +[default0]:[2023-02-16 06:20:02,144] [INFO] [utils.py:827:see_memory_usage] Before Building Model +[default0]:[2023-02-16 06:20:02,144] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB +[default0]:[2023-02-16 06:20:02,145] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.06 GB, percent = 9.0% +[default0]:SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +[default0]:Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data=11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63} +[default0]:[2023-02-16 06:20:03,460] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +[default0]:stage=0 layers=41 +[default0]: 0: _to_float16 +[default0]: 1: EmbeddingPipe +[default0]: 2: +[default0]: 3: +[default0]: 4: ParallelTransformerLayerPipe +[default0]: 5: ParallelTransformerLayerPipe +[default0]: 6: ParallelTransformerLayerPipe +[default0]: 7: ParallelTransformerLayerPipe +[default0]: 8: ParallelTransformerLayerPipe +[default0]: 9: ParallelTransformerLayerPipe +[default0]: 10: ParallelTransformerLayerPipe +[default0]: 11: ParallelTransformerLayerPipe +[default0]: 12: ParallelTransformerLayerPipe +[default0]: 13: ParallelTransformerLayerPipe +[default0]: 14: ParallelTransformerLayerPipe +[default0]: 15: ParallelTransformerLayerPipe +[default0]: 16: ParallelTransformerLayerPipe +[default0]: 17: ParallelTransformerLayerPipe +[default0]: 18: ParallelTransformerLayerPipe +[default0]: 19: ParallelTransformerLayerPipe +[default0]: 20: ParallelTransformerLayerPipe +[default0]: 21: ParallelTransformerLayerPipe +[default0]: 22: ParallelTransformerLayerPipe +[default0]: 23: ParallelTransformerLayerPipe +[default0]: 24: ParallelTransformerLayerPipe +[default0]: 25: ParallelTransformerLayerPipe +[default0]: 26: ParallelTransformerLayerPipe +[default0]: 27: ParallelTransformerLayerPipe +[default0]: 28: ParallelTransformerLayerPipe +[default0]: 29: ParallelTransformerLayerPipe +[default0]: 30: ParallelTransformerLayerPipe +[default0]: 31: ParallelTransformerLayerPipe +[default0]: 32: ParallelTransformerLayerPipe +[default0]: 33: ParallelTransformerLayerPipe +[default0]: 34: ParallelTransformerLayerPipe +[default0]: 35: ParallelTransformerLayerPipe +[default0]: 36: undo +[default0]: 37: MixedFusedLayerNorm +[default0]: 38: +[default0]: 39: EmbeddingPipe +[default0]: 40: float16_to_fp32 +[default0]: loss: CrossEntropy +[default0]:[2023-02-16 06:20:03,667] [INFO] [utils.py:827:see_memory_usage] After Building Model +[default0]:[2023-02-16 06:20:03,667] [INFO] [utils.py:828:see_memory_usage] MA 2.56 GB Max_MA 2.56 GB CA 2.62 GB Max_CA 3 GB +[default0]:[2023-02-16 06:20:03,668] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.59 GB, percent = 9.1% +[default0]:building GPT model ... +[default0]:[2023-02-16 06:20:03,733] [INFO] [utils.py:827:see_memory_usage] Before Building Model +[default0]:[2023-02-16 06:20:03,733] [INFO] [utils.py:828:see_memory_usage] MA 2.56 GB Max_MA 2.56 GB CA 2.62 GB Max_CA 3 GB +[default0]:[2023-02-16 06:20:03,733] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.59 GB, percent = 9.1% +[default0]:SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +[default0]:Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data=11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63} +[default0]:[2023-02-16 06:20:05,049] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +[default0]:stage=0 layers=41 +[default0]: 0: _to_float16 +[default0]: 1: EmbeddingPipe +[default0]: 2: +[default0]: 3: +[default0]: 4: ParallelTransformerLayerPipe +[default0]: 5: ParallelTransformerLayerPipe +[default0]: 6: ParallelTransformerLayerPipe +[default0]: 7: ParallelTransformerLayerPipe +[default0]: 8: ParallelTransformerLayerPipe +[default0]: 9: ParallelTransformerLayerPipe +[default0]: 10: ParallelTransformerLayerPipe +[default0]: 11: ParallelTransformerLayerPipe +[default0]: 12: ParallelTransformerLayerPipe +[default0]: 13: ParallelTransformerLayerPipe +[default0]: 14: ParallelTransformerLayerPipe +[default0]: 15: ParallelTransformerLayerPipe +[default0]: 16: ParallelTransformerLayerPipe +[default0]: 17: ParallelTransformerLayerPipe +[default0]: 18: ParallelTransformerLayerPipe +[default0]: 19: ParallelTransformerLayerPipe +[default0]: 20: ParallelTransformerLayerPipe +[default0]: 21: ParallelTransformerLayerPipe +[default0]: 22: ParallelTransformerLayerPipe +[default0]: 23: ParallelTransformerLayerPipe +[default0]: 24: ParallelTransformerLayerPipe +[default0]: 25: ParallelTransformerLayerPipe +[default0]: 26: ParallelTransformerLayerPipe +[default0]: 27: ParallelTransformerLayerPipe +[default0]: 28: ParallelTransformerLayerPipe +[default0]: 29: ParallelTransformerLayerPipe +[default0]: 30: ParallelTransformerLayerPipe +[default0]: 31: ParallelTransformerLayerPipe +[default0]: 32: ParallelTransformerLayerPipe +[default0]: 33: ParallelTransformerLayerPipe +[default0]: 34: ParallelTransformerLayerPipe +[default0]: 35: ParallelTransformerLayerPipe +[default0]: 36: undo +[default0]: 37: MixedFusedLayerNorm +[default0]: 38: +[default0]: 39: EmbeddingPipe +[default0]: 40: float16_to_fp32 +[default0]: loss: CrossEntropy +[default0]:[2023-02-16 06:20:05,162] [INFO] [utils.py:827:see_memory_usage] After Building Model +[default0]:[2023-02-16 06:20:05,162] [INFO] [utils.py:828:see_memory_usage] MA 2.57 GB Max_MA 2.57 GB CA 2.63 GB Max_CA 3 GB +[default0]:[2023-02-16 06:20:05,162] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.62 GB, percent = 9.1% +[default0]:building GPT model ... +[default0]:[2023-02-16 06:20:05,229] [INFO] [utils.py:827:see_memory_usage] Before Building Model +[default0]:[2023-02-16 06:20:05,230] [INFO] [utils.py:828:see_memory_usage] MA 2.57 GB Max_MA 2.57 GB CA 2.63 GB Max_CA 3 GB +[default0]:[2023-02-16 06:20:05,230] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.62 GB, percent = 9.1% +[default0]:SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None +[default0]:Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data=11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63} +[default0]:[2023-02-16 06:20:06,538] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer +[default0]:stage=0 layers=41 +[default0]: 0: _to_float16 +[default0]: 1: EmbeddingPipe +[default0]: 2: +[default0]: 3: +[default0]: 4: ParallelTransformerLayerPipe +[default0]: 5: ParallelTransformerLayerPipe +[default0]: 6: ParallelTransformerLayerPipe +[default0]: 7: ParallelTransformerLayerPipe +[default0]: 8: ParallelTransformerLayerPipe +[default0]: 9: ParallelTransformerLayerPipe +[default0]: 10: ParallelTransformerLayerPipe +[default0]: 11: ParallelTransformerLayerPipe +[default0]: 12: ParallelTransformerLayerPipe +[default0]: 13: ParallelTransformerLayerPipe +[default0]: 14: ParallelTransformerLayerPipe +[default0]: 15: ParallelTransformerLayerPipe +[default0]: 16: ParallelTransformerLayerPipe +[default0]: 17: ParallelTransformerLayerPipe +[default0]: 18: ParallelTransformerLayerPipe +[default0]: 19: ParallelTransformerLayerPipe +[default0]: 20: ParallelTransformerLayerPipe +[default0]: 21: ParallelTransformerLayerPipe +[default0]: 22: ParallelTransformerLayerPipe +[default0]: 23: ParallelTransformerLayerPipe +[default0]: 24: ParallelTransformerLayerPipe +[default0]: 25: ParallelTransformerLayerPipe +[default0]: 26: ParallelTransformerLayerPipe +[default0]: 27: ParallelTransformerLayerPipe +[default0]: 28: ParallelTransformerLayerPipe +[default0]: 29: ParallelTransformerLayerPipe +[default0]: 30: ParallelTransformerLayerPipe +[default0]: 31: ParallelTransformerLayerPipe +[default0]: 32: ParallelTransformerLayerPipe +[default0]: 33: ParallelTransformerLayerPipe +[default0]: 34: ParallelTransformerLayerPipe +[default0]: 35: ParallelTransformerLayerPipe +[default0]: 36: undo +[default0]: 37: MixedFusedLayerNorm +[default0]: 38: +[default0]: 39: EmbeddingPipe +[default0]: 40: float16_to_fp32 +[default0]: loss: CrossEntropy +[default0]:[2023-02-16 06:20:06,669] [INFO] [utils.py:827:see_memory_usage] After Building Model +[default0]:[2023-02-16 06:20:06,670] [INFO] [utils.py:828:see_memory_usage] MA 2.59 GB Max_MA 2.59 GB CA 2.66 GB Max_CA 3 GB +[default0]:[2023-02-16 06:20:06,670] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 45.63 GB, percent = 9.1% +[default0]:setting training iterations to 3814 +[default0]:> learning rate decay style: cosine +[default0]:DeepSpeed is enabled. +[default0]:[2023-02-16 06:20:06,706] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.2+2a644488, git-hash=2a644488, git-branch=master +[default0]:[2023-02-16 06:20:08,236] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +[default0]:[2023-02-16 06:20:08,237] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer +[default0]:[2023-02-16 06:20:08,237] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer +[default0]:[2023-02-16 06:20:08,249] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = {basic_optimizer.__class__.__name__} +[default0]:[2023-02-16 06:20:08,249] [INFO] [utils.py:52:is_zero_supported_optimizer] Checking ZeRO support for optimizer=FusedAdam type= +[default0]:[2023-02-16 06:20:08,249] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 1 optimizer +[default0]:[2023-02-16 06:20:08,249] [INFO] [stage_1_and_2.py:134:__init__] Reduce bucket size 500000000 +[default0]:[2023-02-16 06:20:08,249] [INFO] [stage_1_and_2.py:135:__init__] Allgather bucket size 500000000 +[default0]:[2023-02-16 06:20:08,249] [INFO] [stage_1_and_2.py:136:__init__] CPU Offload: False +[default0]:[2023-02-16 06:20:08,249] [INFO] [stage_1_and_2.py:137:__init__] Round robin gradient partitioning: False +[default6]:Rank: 62 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 16 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 4 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 50 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 24 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 30 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 32 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 26 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 6 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 0 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 34 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 18 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 22 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 14 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 2 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 54 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 48 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 40 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 58 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 56 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 20 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 10 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:Rank: 8 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 46 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 52 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default6]:Rank: 38 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default2]:Rank: 42 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 12 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 36 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 44 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 7 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 51 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 29 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 19 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 21 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 55 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 59 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 1 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 13 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 9 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 41 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 57 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 60 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 45 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 35 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 39 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default4]:Rank: 28 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 11 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 15 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 47 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 5 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 49 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 53 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 17 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 43 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 23 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 3 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 33 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 37 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default5]:Rank: 61 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 63 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default1]:Rank: 25 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default7]:Rank: 31 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default3]:Rank: 27 partition count [32, 32, 32, 32, 32, 32] and sizes[(4915200, False), (14336000, False), (2048000, False), (15155200, False), (4915200, False), (24640, False)] +[default0]:[2023-02-16 06:20:12,609] [INFO] [utils.py:827:see_memory_usage] Before initializing optimizer states +[default0]:[2023-02-16 06:20:12,609] [INFO] [utils.py:828:see_memory_usage] MA 2.66 GB Max_MA 2.69 GB CA 5.17 GB Max_CA 5 GB +[default0]:[2023-02-16 06:20:12,609] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 60.72 GB, percent = 12.1% +[default0]:[2023-02-16 06:20:12,726] [INFO] [utils.py:827:see_memory_usage] After initializing optimizer states +[default0]:[2023-02-16 06:20:12,727] [INFO] [utils.py:828:see_memory_usage] MA 2.96 GB Max_MA 3.12 GB CA 5.5 GB Max_CA 6 GB +[default0]:[2023-02-16 06:20:12,727] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 60.72 GB, percent = 12.1% +[default0]:[2023-02-16 06:20:12,727] [INFO] [stage_1_and_2.py:516:__init__] optimizer state initialized +[default0]:[2023-02-16 06:20:12,794] [INFO] [utils.py:827:see_memory_usage] After initializing ZeRO optimizer +[default0]:[2023-02-16 06:20:12,794] [INFO] [utils.py:828:see_memory_usage] MA 2.96 GB Max_MA 2.96 GB CA 5.5 GB Max_CA 6 GB +[default0]:[2023-02-16 06:20:12,794] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 60.74 GB, percent = 12.1% +[default0]:[2023-02-16 06:20:12,795] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +[default0]:[2023-02-16 06:20:12,795] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler +[default0]:[2023-02-16 06:20:12,795] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +[default0]:[2023-02-16 06:20:12,795] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0, 0.0, 0.0, 0.0, 0.0, 0.0], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95)] +[default0]:[2023-02-16 06:20:12,795] [INFO] [config.py:987:print] DeepSpeedEngine configuration: +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] activation_checkpointing_config { +[default0]: "partition_activations": false, +[default0]: "contiguous_memory_optimization": false, +[default0]: "cpu_checkpointing": false, +[default0]: "number_checkpoints": null, +[default0]: "synchronize_checkpoint_boundary": false, +[default0]: "profile": false +[default0]:} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] amp_enabled .................. False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] amp_params ................... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] autotuning_config ............ { +[default0]: "enabled": false, +[default0]: "start_step": null, +[default0]: "end_step": null, +[default0]: "metric_path": null, +[default0]: "arg_mappings": null, +[default0]: "metric": "throughput", +[default0]: "model_info": null, +[default0]: "results_dir": null, +[default0]: "exps_dir": null, +[default0]: "overwrite": true, +[default0]: "fast": true, +[default0]: "start_profile_step": 3, +[default0]: "end_profile_step": 5, +[default0]: "tuner_type": "gridsearch", +[default0]: "tuner_early_stopping": 5, +[default0]: "tuner_num_trials": 50, +[default0]: "model_info_path": null, +[default0]: "mp_size": 1, +[default0]: "max_train_batch_size": null, +[default0]: "min_train_batch_size": 1, +[default0]: "max_train_micro_batch_size_per_gpu": 1.024000e+03, +[default0]: "min_train_micro_batch_size_per_gpu": 1, +[default0]: "num_tuning_micro_batch_sizes": 3 +[default0]:} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] bfloat16_enabled ............. False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] checkpoint_tag_validation_enabled True +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] checkpoint_tag_validation_fail False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] comms_config ................. +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] communication_data_type ...... None +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] curriculum_enabled ........... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] curriculum_params ............ False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] dataloader_drop_last ......... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] disable_allgather ............ False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] dump_state ................... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] dynamic_loss_scale_args ...... {'init_scale': 4096, 'scale_window': 500, 'delayed_shift': 2, 'min_scale': 1} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_enabled ........... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_gas_boundary_resolution 1 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_layer_name ........ bert.encoder.layer +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_layer_num ......... 0 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_max_iter .......... 100 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_stability ......... 1e-06 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_tol ............... 0.01 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] eigenvalue_verbose ........... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] elasticity_enabled ........... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] flops_profiler_config ........ { +[default0]: "enabled": false, +[default0]: "profile_step": 1, +[default0]: "module_depth": -1, +[default0]: "top_modules": 1, +[default0]: "detailed": true, +[default0]: "output_file": null +[default0]:} +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] fp16_auto_cast ............... False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] fp16_enabled ................. True +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] fp16_master_weights_and_gradients False +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] global_rank .................. 0 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] gradient_accumulation_steps .. 1 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] gradient_clipping ............ 1.0 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] gradient_predivide_factor .... 1.0 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] initial_dynamic_scale ........ 4096 +[default0]:[2023-02-16 06:20:12,796] [INFO] [config.py:991:print] load_universal_checkpoint .... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] loss_scale ................... 0 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] memory_breakdown ............. False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] monitor_config ............... +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] nebula_config ................ { +[default0]: "enabled": false, +[default0]: "persistent_storage_path": null, +[default0]: "persistent_time_interval": 100, +[default0]: "num_of_version_in_retention": 2, +[default0]: "enable_nebula_load": true, +[default0]: "load_path": null +[default0]:} +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] optimizer_legacy_fusion ...... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] optimizer_name ............... None +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] optimizer_params ............. None +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] pld_enabled .................. False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] pld_params ................... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] prescale_gradients ........... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] scheduler_name ............... None +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] scheduler_params ............. None +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] sparse_attention ............. None +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] sparse_gradients_enabled ..... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] steps_per_print .............. 2000 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] train_batch_size ............. 512 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] train_micro_batch_size_per_gpu 16 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] wall_clock_breakdown ......... False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] world_size ................... 32 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] zero_allow_untested_optimizer False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] zero_config .................. stage=1 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] zero_enabled ................. True +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:991:print] zero_optimization_stage ...... 1 +[default0]:[2023-02-16 06:20:12,797] [INFO] [config.py:976:print_user_config] json = { +[default0]: "train_micro_batch_size_per_gpu": 16, +[default0]: "train_batch_size": 512, +[default0]: "gradient_clipping": 1.0, +[default0]: "zero_optimization": { +[default0]: "stage": 1 +[default0]: }, +[default0]: "fp16": { +[default0]: "enabled": true, +[default0]: "loss_scale": 0, +[default0]: "loss_scale_window": 500, +[default0]: "hysteresis": 2, +[default0]: "min_loss_scale": 1, +[default0]: "initial_scale_power": 12 +[default0]: }, +[default0]: "steps_per_print": 2.000000e+03, +[default0]: "wall_clock_breakdown": false +[default0]:} +[default0]:[2023-02-16 06:20:12,797] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=1 micro_batch_size=16 +[default0]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:time (ms) | load-checkpoint: 9.92 +[default0]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,050] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1324615680 (1324.616M) TOTAL_PARAMS=2649231360 (2649.231M) UNIQUE_PARAMS=2649231360 (2649.231M) +[default1]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:[2023-02-16 06:20:13,050] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1324615680 (1324.616M) TOTAL_PARAMS=2649231360 (2649.231M) UNIQUE_PARAMS=2649231360 (2649.231M) +[default0]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:WARNING: could not find the metadata file /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]: will not load any checkpoints and will start from random +[default0]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default5]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default2]:[2023-02-16 06:20:13,091] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,087] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default3]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default4]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default1]:[2023-02-16 06:20:13,083] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default6]:[2023-02-16 06:20:13,088] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default7]:[2023-02-16 06:20:13,090] [WARNING] [engine.py:2574:load_checkpoint] Unable to find latest file at /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[default0]:estimated model parameters: 2.64923136 +[default0]:estimated model parameters without embeddings: 2.51815936 +[default0]:/gpfsdswork/projects/rech/ajs/commun/code/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings +[default0]: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") +[default0]:[after model, optimizer, and learning rate scheduler are built] datetime: 2023-02-16 06:20:13 +[default0]:> building train, validation, and test datasets ... +[default0]: > datasets target sizes (minimum size): +[default0]: train: 1953125 +[default0]: validation: 2048 +[default0]: test: 512 +[default0]:> building train, validation, and test datasets for GPT ... +[default0]: > building dataset index ... +[default0]: reading sizes... +[default0]: reading pointers... +[default0]: reading document index... +[default0]: creating numpy buffer of mmap... +[default0]: creating memory view of numpy buffer... +[default0]: > finished creating indexed dataset in 0.057539 seconds +[default0]: number of documents: 364868892 +[default0]: > dataset split: +[default0]: train: +[default0]: document indices in [0, 353557956) total of 353557956 documents +[default0]: validation: +[default0]: document indices in [353557956, 364504023) total of 10946067 documents +[default0]: test: +[default0]: document indices in [364504023, 364868892) total of 364869 documents +[default0]: > loading doc-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_train_indexmap_1953125ns_2048sl_42s_doc_idx.npy +[default0]: > loading sample-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_train_indexmap_1953125ns_2048sl_42s_sample_idx.npy +[default0]: > loading shuffle-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_train_indexmap_1953125ns_2048sl_42s_shuffle_idx.npy +[default0]: loaded indexed file in 0.036 seconds +[default0]: total number of samples: 82627993 +[default0]: total number of epochs: 1 +[default0]: > loading doc-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_valid_indexmap_2048ns_2048sl_42s_doc_idx.npy +[default0]: > loading sample-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_valid_indexmap_2048ns_2048sl_42s_sample_idx.npy +[default0]: > loading shuffle-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_valid_indexmap_2048ns_2048sl_42s_shuffle_idx.npy +[default0]: loaded indexed file in 0.035 seconds +[default0]: total number of samples: 2558380 +[default0]: total number of epochs: 1 +[default0]: > loading doc-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_test_indexmap_512ns_2048sl_42s_doc_idx.npy +[default0]: > loading sample-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_test_indexmap_512ns_2048sl_42s_sample_idx.npy +[default0]: > loading shuffle-idx mapping from /gpfsscratch/rech/ajs/commun/datasets/c4/gpt2tok_c4_text_document_test_indexmap_512ns_2048sl_42s_shuffle_idx.npy +[default0]: loaded indexed file in 0.007 seconds +[default0]: total number of samples: 84955 +[default0]: total number of epochs: 1 +[default0]:> finished creating GPT datasets ... +[default0]:[after dataloaders are built] datetime: 2023-02-16 06:20:22 +[default0]:done with setup ... +[default0]:training ... +[default0]:Number of parameters: [tensor rank - pipeline rank] w/ and w/o embeddings: +[default0]:[000-000] 2.6492B / 2.5182B +[default7]:time (ms) | model-and-optimizer-setup: 11262.32 | train/valid/test-data-iterators-setup: 7346.41 +[default1]:[001-000] 2.6492B / 2.5182B +[default0]:[before the start of training step] datetime: 2023-02-16 06:20:22 +[default0]:[2023-02-16 06:20:22,951] [INFO] [checkpointing.py:547:forward] Activation Checkpointing Information +[default0]:[2023-02-16 06:20:22,951] [INFO] [checkpointing.py:548:forward] ----Partition Activations False, CPU CHECKPOINTING False +[default0]:[2023-02-16 06:20:22,951] [INFO] [checkpointing.py:551:forward] ----contiguous Memory Checkpointing False with 32 total layers +[default0]:[2023-02-16 06:20:22,951] [INFO] [checkpointing.py:554:forward] ----Synchronization False +[default0]:[2023-02-16 06:20:22,951] [INFO] [checkpointing.py:555:forward] ----Profiling time in checkpointing False +[default0]:[Rank 0] (after 1 iterations) memory (MB) | allocated: 6240.66552734375 | max allocated: 20946.41015625 | reserved: 24996.0 | max reserved: 24996.0 +[default1]:[Rank 1] (after 1 iterations) memory (MB) | allocated: 6264.68505859375 | max allocated: 20970.4296875 | reserved: 24998.0 | max reserved: 24998.0 +[default7]: iteration 1/ 3814 | consumed samples: 512 | consumed tokens: 1048576 | elapsed time per iteration (s): 6.13 | learning rate: 2.796E-06 | global batch size: 512 | lm loss: 1.084328E+01 | loss scale: 4096.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 83.517 | TFLOPs: 63.08 | +[default7]: iteration 2/ 3814 | consumed samples: 1024 | consumed tokens: 2097152 | elapsed time per iteration (s): 5.35 | learning rate: 5.592E-06 | global batch size: 512 | lm loss: 1.084320E+01 | loss scale: 4096.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.671 | TFLOPs: 72.26 | +[default7]: iteration 3/ 3814 | consumed samples: 1536 | consumed tokens: 3145728 | elapsed time per iteration (s): 5.34 | learning rate: 8.389E-06 | global batch size: 512 | lm loss: 1.084254E+01 | loss scale: 4096.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.907 | TFLOPs: 72.44 | +[default7]: iteration 4/ 3814 | consumed samples: 2048 | consumed tokens: 4194304 | elapsed time per iteration (s): 5.35 | learning rate: 1.118E-05 | global batch size: 512 | lm loss: 1.083950E+01 | loss scale: 4096.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.698 | TFLOPs: 72.28 | +[default7]: iteration 5/ 3814 | consumed samples: 2560 | consumed tokens: 5242880 | elapsed time per iteration (s): 5.37 | learning rate: 1.398E-05 | global batch size: 512 | lm loss: 1.082619E+01 | loss scale: 4096.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.417 | TFLOPs: 72.07 | +[default7]: iteration 6/ 3814 | consumed samples: 3072 | consumed tokens: 6291456 | elapsed time per iteration (s): 5.33 | learning rate: 1.678E-05 | global batch size: 512 | lm loss: 1.081479E+01 | loss scale: 4096.0 | grad norm: 0.063 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.120 | TFLOPs: 72.60 | +[default7]: iteration 7/ 3814 | consumed samples: 3584 | consumed tokens: 7340032 | elapsed time per iteration (s): 5.35 | learning rate: 1.957E-05 | global batch size: 512 | lm loss: 1.080501E+01 | loss scale: 4096.0 | grad norm: 0.051 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.707 | TFLOPs: 72.29 | +[default7]: iteration 8/ 3814 | consumed samples: 4096 | consumed tokens: 8388608 | elapsed time per iteration (s): 5.35 | learning rate: 2.237E-05 | global batch size: 512 | lm loss: 1.079552E+01 | loss scale: 4096.0 | grad norm: 0.044 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.640 | TFLOPs: 72.24 | +[default7]: iteration 9/ 3814 | consumed samples: 4608 | consumed tokens: 9437184 | elapsed time per iteration (s): 5.38 | learning rate: 2.517E-05 | global batch size: 512 | lm loss: 1.078986E+01 | loss scale: 4096.0 | grad norm: 0.039 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.228 | TFLOPs: 71.93 | +[default7]: iteration 10/ 3814 | consumed samples: 5120 | consumed tokens: 10485760 | elapsed time per iteration (s): 5.36 | learning rate: 2.796E-05 | global batch size: 512 | lm loss: 1.078151E+01 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.573 | TFLOPs: 72.19 | +[default7]: iteration 11/ 3814 | consumed samples: 5632 | consumed tokens: 11534336 | elapsed time per iteration (s): 5.33 | learning rate: 3.076E-05 | global batch size: 512 | lm loss: 1.077419E+01 | loss scale: 4096.0 | grad norm: 0.030 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.025 | TFLOPs: 72.53 | +[default7]: iteration 12/ 3814 | consumed samples: 6144 | consumed tokens: 12582912 | elapsed time per iteration (s): 5.35 | learning rate: 3.355E-05 | global batch size: 512 | lm loss: 1.076853E+01 | loss scale: 4096.0 | grad norm: 0.027 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.716 | TFLOPs: 72.30 | +[default7]: iteration 13/ 3814 | consumed samples: 6656 | consumed tokens: 13631488 | elapsed time per iteration (s): 5.32 | learning rate: 3.635E-05 | global batch size: 512 | lm loss: 1.076295E+01 | loss scale: 4096.0 | grad norm: 0.024 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.329 | TFLOPs: 72.76 | +[default7]: iteration 14/ 3814 | consumed samples: 7168 | consumed tokens: 14680064 | elapsed time per iteration (s): 5.32 | learning rate: 3.915E-05 | global batch size: 512 | lm loss: 1.075664E+01 | loss scale: 4096.0 | grad norm: 0.022 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.166 | TFLOPs: 72.63 | +[default7]: iteration 15/ 3814 | consumed samples: 7680 | consumed tokens: 15728640 | elapsed time per iteration (s): 5.32 | learning rate: 4.194E-05 | global batch size: 512 | lm loss: 1.075213E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.194 | TFLOPs: 72.66 | +[default7]: iteration 16/ 3814 | consumed samples: 8192 | consumed tokens: 16777216 | elapsed time per iteration (s): 5.34 | learning rate: 4.474E-05 | global batch size: 512 | lm loss: 1.074667E+01 | loss scale: 4096.0 | grad norm: 0.020 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.874 | TFLOPs: 72.41 | +[default7]: iteration 17/ 3814 | consumed samples: 8704 | consumed tokens: 17825792 | elapsed time per iteration (s): 5.35 | learning rate: 4.754E-05 | global batch size: 512 | lm loss: 1.074171E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.768 | TFLOPs: 72.33 | +[default7]: iteration 18/ 3814 | consumed samples: 9216 | consumed tokens: 18874368 | elapsed time per iteration (s): 5.33 | learning rate: 5.033E-05 | global batch size: 512 | lm loss: 1.073662E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.134 | TFLOPs: 72.61 | +[default7]: iteration 19/ 3814 | consumed samples: 9728 | consumed tokens: 19922944 | elapsed time per iteration (s): 5.30 | learning rate: 5.313E-05 | global batch size: 512 | lm loss: 1.073207E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.550 | TFLOPs: 72.93 | +[default7]: iteration 20/ 3814 | consumed samples: 10240 | consumed tokens: 20971520 | elapsed time per iteration (s): 5.32 | learning rate: 5.592E-05 | global batch size: 512 | lm loss: 1.072633E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.200 | TFLOPs: 72.66 | +[default7]: iteration 21/ 3814 | consumed samples: 10752 | consumed tokens: 22020096 | elapsed time per iteration (s): 5.31 | learning rate: 5.872E-05 | global batch size: 512 | lm loss: 1.072130E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.419 | TFLOPs: 72.83 | +[default7]: iteration 22/ 3814 | consumed samples: 11264 | consumed tokens: 23068672 | elapsed time per iteration (s): 5.33 | learning rate: 6.152E-05 | global batch size: 512 | lm loss: 1.071530E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.065 | TFLOPs: 72.56 | +[default7]: iteration 23/ 3814 | consumed samples: 11776 | consumed tokens: 24117248 | elapsed time per iteration (s): 5.30 | learning rate: 6.431E-05 | global batch size: 512 | lm loss: 1.070891E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.524 | TFLOPs: 72.91 | +[default7]: iteration 24/ 3814 | consumed samples: 12288 | consumed tokens: 25165824 | elapsed time per iteration (s): 5.31 | learning rate: 6.711E-05 | global batch size: 512 | lm loss: 1.070145E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.382 | TFLOPs: 72.80 | +[default7]: iteration 25/ 3814 | consumed samples: 12800 | consumed tokens: 26214400 | elapsed time per iteration (s): 5.31 | learning rate: 6.991E-05 | global batch size: 512 | lm loss: 1.069491E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.385 | TFLOPs: 72.80 | +[default7]: iteration 26/ 3814 | consumed samples: 13312 | consumed tokens: 27262976 | elapsed time per iteration (s): 5.31 | learning rate: 7.270E-05 | global batch size: 512 | lm loss: 1.068623E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.393 | TFLOPs: 72.81 | +[default7]: iteration 27/ 3814 | consumed samples: 13824 | consumed tokens: 28311552 | elapsed time per iteration (s): 5.31 | learning rate: 7.550E-05 | global batch size: 512 | lm loss: 1.067869E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.428 | TFLOPs: 72.83 | +[default7]: iteration 28/ 3814 | consumed samples: 14336 | consumed tokens: 29360128 | elapsed time per iteration (s): 5.32 | learning rate: 7.829E-05 | global batch size: 512 | lm loss: 1.067131E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.168 | TFLOPs: 72.64 | +[default7]: iteration 29/ 3814 | consumed samples: 14848 | consumed tokens: 30408704 | elapsed time per iteration (s): 5.31 | learning rate: 8.109E-05 | global batch size: 512 | lm loss: 1.066200E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.335 | TFLOPs: 72.76 | +[default7]: iteration 30/ 3814 | consumed samples: 15360 | consumed tokens: 31457280 | elapsed time per iteration (s): 5.32 | learning rate: 8.389E-05 | global batch size: 512 | lm loss: 1.065213E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.216 | TFLOPs: 72.67 | +[default7]: iteration 31/ 3814 | consumed samples: 15872 | consumed tokens: 32505856 | elapsed time per iteration (s): 5.31 | learning rate: 8.668E-05 | global batch size: 512 | lm loss: 1.064308E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.408 | TFLOPs: 72.82 | +[default7]: iteration 32/ 3814 | consumed samples: 16384 | consumed tokens: 33554432 | elapsed time per iteration (s): 5.31 | learning rate: 8.948E-05 | global batch size: 512 | lm loss: 1.063244E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.406 | TFLOPs: 72.82 | +[default7]: iteration 33/ 3814 | consumed samples: 16896 | consumed tokens: 34603008 | elapsed time per iteration (s): 5.32 | learning rate: 9.227E-05 | global batch size: 512 | lm loss: 1.062215E+01 | loss scale: 4096.0 | grad norm: 0.018 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.283 | TFLOPs: 72.72 | +[default7]: iteration 34/ 3814 | consumed samples: 17408 | consumed tokens: 35651584 | elapsed time per iteration (s): 5.30 | learning rate: 9.507E-05 | global batch size: 512 | lm loss: 1.061195E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.560 | TFLOPs: 72.93 | +[default7]: iteration 35/ 3814 | consumed samples: 17920 | consumed tokens: 36700160 | elapsed time per iteration (s): 5.30 | learning rate: 9.787E-05 | global batch size: 512 | lm loss: 1.059945E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.586 | TFLOPs: 72.95 | +[default7]: iteration 36/ 3814 | consumed samples: 18432 | consumed tokens: 37748736 | elapsed time per iteration (s): 5.30 | learning rate: 1.007E-04 | global batch size: 512 | lm loss: 1.058723E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.677 | TFLOPs: 73.02 | +[default7]: iteration 37/ 3814 | consumed samples: 18944 | consumed tokens: 38797312 | elapsed time per iteration (s): 5.32 | learning rate: 1.035E-04 | global batch size: 512 | lm loss: 1.057483E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.189 | TFLOPs: 72.65 | +[default7]: iteration 38/ 3814 | consumed samples: 19456 | consumed tokens: 39845888 | elapsed time per iteration (s): 5.31 | learning rate: 1.063E-04 | global batch size: 512 | lm loss: 1.056031E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.509 | TFLOPs: 72.89 | +[default7]: iteration 39/ 3814 | consumed samples: 19968 | consumed tokens: 40894464 | elapsed time per iteration (s): 5.31 | learning rate: 1.091E-04 | global batch size: 512 | lm loss: 1.054804E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.373 | TFLOPs: 72.79 | +[default7]: iteration 40/ 3814 | consumed samples: 20480 | consumed tokens: 41943040 | elapsed time per iteration (s): 5.32 | learning rate: 1.118E-04 | global batch size: 512 | lm loss: 1.053356E+01 | loss scale: 4096.0 | grad norm: 0.019 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.303 | TFLOPs: 72.74 | +[default7]: iteration 41/ 3814 | consumed samples: 20992 | consumed tokens: 42991616 | elapsed time per iteration (s): 5.31 | learning rate: 1.146E-04 | global batch size: 512 | lm loss: 1.051729E+01 | loss scale: 4096.0 | grad norm: 0.020 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.486 | TFLOPs: 72.88 | +[default7]: iteration 42/ 3814 | consumed samples: 21504 | consumed tokens: 44040192 | elapsed time per iteration (s): 5.31 | learning rate: 1.174E-04 | global batch size: 512 | lm loss: 1.050442E+01 | loss scale: 4096.0 | grad norm: 0.020 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.423 | TFLOPs: 72.83 | +[default7]: iteration 43/ 3814 | consumed samples: 22016 | consumed tokens: 45088768 | elapsed time per iteration (s): 5.31 | learning rate: 1.202E-04 | global batch size: 512 | lm loss: 1.048767E+01 | loss scale: 4096.0 | grad norm: 0.020 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.376 | TFLOPs: 72.79 | +[default7]: iteration 44/ 3814 | consumed samples: 22528 | consumed tokens: 46137344 | elapsed time per iteration (s): 5.31 | learning rate: 1.230E-04 | global batch size: 512 | lm loss: 1.047475E+01 | loss scale: 4096.0 | grad norm: 0.020 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.350 | TFLOPs: 72.77 | +[default7]: iteration 45/ 3814 | consumed samples: 23040 | consumed tokens: 47185920 | elapsed time per iteration (s): 5.33 | learning rate: 1.258E-04 | global batch size: 512 | lm loss: 1.045738E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.979 | TFLOPs: 72.49 | +[default7]: iteration 46/ 3814 | consumed samples: 23552 | consumed tokens: 48234496 | elapsed time per iteration (s): 5.31 | learning rate: 1.286E-04 | global batch size: 512 | lm loss: 1.044127E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.420 | TFLOPs: 72.83 | +[default7]: iteration 47/ 3814 | consumed samples: 24064 | consumed tokens: 49283072 | elapsed time per iteration (s): 5.31 | learning rate: 1.314E-04 | global batch size: 512 | lm loss: 1.042356E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.373 | TFLOPs: 72.79 | +[default7]: iteration 48/ 3814 | consumed samples: 24576 | consumed tokens: 50331648 | elapsed time per iteration (s): 5.33 | learning rate: 1.342E-04 | global batch size: 512 | lm loss: 1.040697E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.042 | TFLOPs: 72.54 | +[default7]: iteration 49/ 3814 | consumed samples: 25088 | consumed tokens: 51380224 | elapsed time per iteration (s): 5.31 | learning rate: 1.370E-04 | global batch size: 512 | lm loss: 1.038799E+01 | loss scale: 4096.0 | grad norm: 0.021 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.376 | TFLOPs: 72.79 | +[default7]: iteration 50/ 3814 | consumed samples: 25600 | consumed tokens: 52428800 | elapsed time per iteration (s): 5.32 | learning rate: 1.398E-04 | global batch size: 512 | lm loss: 1.037067E+01 | loss scale: 4096.0 | grad norm: 0.022 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.246 | TFLOPs: 72.70 | +[default7]: iteration 51/ 3814 | consumed samples: 26112 | consumed tokens: 53477376 | elapsed time per iteration (s): 5.32 | learning rate: 1.426E-04 | global batch size: 512 | lm loss: 1.035166E+01 | loss scale: 4096.0 | grad norm: 0.022 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.256 | TFLOPs: 72.70 | +[default7]: iteration 52/ 3814 | consumed samples: 26624 | consumed tokens: 54525952 | elapsed time per iteration (s): 5.32 | learning rate: 1.454E-04 | global batch size: 512 | lm loss: 1.033267E+01 | loss scale: 4096.0 | grad norm: 0.022 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.185 | TFLOPs: 72.65 | +[default7]: iteration 53/ 3814 | consumed samples: 27136 | consumed tokens: 55574528 | elapsed time per iteration (s): 5.32 | learning rate: 1.482E-04 | global batch size: 512 | lm loss: 1.031276E+01 | loss scale: 4096.0 | grad norm: 0.023 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.232 | TFLOPs: 72.69 | +[default7]: iteration 54/ 3814 | consumed samples: 27648 | consumed tokens: 56623104 | elapsed time per iteration (s): 5.31 | learning rate: 1.510E-04 | global batch size: 512 | lm loss: 1.029210E+01 | loss scale: 4096.0 | grad norm: 0.023 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.443 | TFLOPs: 72.84 | +[default7]: iteration 55/ 3814 | consumed samples: 28160 | consumed tokens: 57671680 | elapsed time per iteration (s): 5.32 | learning rate: 1.538E-04 | global batch size: 512 | lm loss: 1.027450E+01 | loss scale: 4096.0 | grad norm: 0.023 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.271 | TFLOPs: 72.71 | +[default7]: iteration 56/ 3814 | consumed samples: 28672 | consumed tokens: 58720256 | elapsed time per iteration (s): 5.32 | learning rate: 1.566E-04 | global batch size: 512 | lm loss: 1.024986E+01 | loss scale: 4096.0 | grad norm: 0.024 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.312 | TFLOPs: 72.75 | +[default7]: iteration 57/ 3814 | consumed samples: 29184 | consumed tokens: 59768832 | elapsed time per iteration (s): 5.33 | learning rate: 1.594E-04 | global batch size: 512 | lm loss: 1.022789E+01 | loss scale: 4096.0 | grad norm: 0.024 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.055 | TFLOPs: 72.55 | +[default7]: iteration 58/ 3814 | consumed samples: 29696 | consumed tokens: 60817408 | elapsed time per iteration (s): 5.32 | learning rate: 1.622E-04 | global batch size: 512 | lm loss: 1.020507E+01 | loss scale: 4096.0 | grad norm: 0.024 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.247 | TFLOPs: 72.70 | +[default7]: iteration 59/ 3814 | consumed samples: 30208 | consumed tokens: 61865984 | elapsed time per iteration (s): 5.31 | learning rate: 1.650E-04 | global batch size: 512 | lm loss: 1.018226E+01 | loss scale: 4096.0 | grad norm: 0.025 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.414 | TFLOPs: 72.82 | +[default7]: iteration 60/ 3814 | consumed samples: 30720 | consumed tokens: 62914560 | elapsed time per iteration (s): 5.31 | learning rate: 1.678E-04 | global batch size: 512 | lm loss: 1.016050E+01 | loss scale: 4096.0 | grad norm: 0.025 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.361 | TFLOPs: 72.78 | +[default7]: iteration 61/ 3814 | consumed samples: 31232 | consumed tokens: 63963136 | elapsed time per iteration (s): 5.31 | learning rate: 1.706E-04 | global batch size: 512 | lm loss: 1.013788E+01 | loss scale: 4096.0 | grad norm: 0.028 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.339 | TFLOPs: 72.77 | +[default7]: iteration 62/ 3814 | consumed samples: 31744 | consumed tokens: 65011712 | elapsed time per iteration (s): 5.31 | learning rate: 1.734E-04 | global batch size: 512 | lm loss: 1.011663E+01 | loss scale: 4096.0 | grad norm: 0.029 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.383 | TFLOPs: 72.80 | +[default7]: iteration 63/ 3814 | consumed samples: 32256 | consumed tokens: 66060288 | elapsed time per iteration (s): 5.31 | learning rate: 1.762E-04 | global batch size: 512 | lm loss: 1.008663E+01 | loss scale: 4096.0 | grad norm: 0.027 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.463 | TFLOPs: 72.86 | +[default7]: iteration 64/ 3814 | consumed samples: 32768 | consumed tokens: 67108864 | elapsed time per iteration (s): 5.31 | learning rate: 1.790E-04 | global batch size: 512 | lm loss: 1.006730E+01 | loss scale: 4096.0 | grad norm: 0.028 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.337 | TFLOPs: 72.76 | +[default7]: iteration 65/ 3814 | consumed samples: 33280 | consumed tokens: 68157440 | elapsed time per iteration (s): 5.32 | learning rate: 1.818E-04 | global batch size: 512 | lm loss: 1.004590E+01 | loss scale: 4096.0 | grad norm: 0.027 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.270 | TFLOPs: 72.71 | +[default7]: iteration 66/ 3814 | consumed samples: 33792 | consumed tokens: 69206016 | elapsed time per iteration (s): 5.30 | learning rate: 1.845E-04 | global batch size: 512 | lm loss: 1.001897E+01 | loss scale: 4096.0 | grad norm: 0.028 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.592 | TFLOPs: 72.96 | +[default7]: iteration 67/ 3814 | consumed samples: 34304 | consumed tokens: 70254592 | elapsed time per iteration (s): 5.30 | learning rate: 1.873E-04 | global batch size: 512 | lm loss: 9.991140E+00 | loss scale: 4096.0 | grad norm: 0.027 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.549 | TFLOPs: 72.92 | +[default7]: iteration 68/ 3814 | consumed samples: 34816 | consumed tokens: 71303168 | elapsed time per iteration (s): 5.33 | learning rate: 1.901E-04 | global batch size: 512 | lm loss: 9.965159E+00 | loss scale: 4096.0 | grad norm: 0.028 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.996 | TFLOPs: 72.51 | +[default7]: iteration 69/ 3814 | consumed samples: 35328 | consumed tokens: 72351744 | elapsed time per iteration (s): 5.31 | learning rate: 1.929E-04 | global batch size: 512 | lm loss: 9.934973E+00 | loss scale: 4096.0 | grad norm: 0.028 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.468 | TFLOPs: 72.86 | +[default7]: iteration 70/ 3814 | consumed samples: 35840 | consumed tokens: 73400320 | elapsed time per iteration (s): 5.30 | learning rate: 1.957E-04 | global batch size: 512 | lm loss: 9.917343E+00 | loss scale: 4096.0 | grad norm: 0.029 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.640 | TFLOPs: 72.99 | +[default7]: iteration 71/ 3814 | consumed samples: 36352 | consumed tokens: 74448896 | elapsed time per iteration (s): 5.31 | learning rate: 1.985E-04 | global batch size: 512 | lm loss: 9.887914E+00 | loss scale: 4096.0 | grad norm: 0.029 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.364 | TFLOPs: 72.78 | +[default7]: iteration 72/ 3814 | consumed samples: 36864 | consumed tokens: 75497472 | elapsed time per iteration (s): 5.31 | learning rate: 2.013E-04 | global batch size: 512 | lm loss: 9.859125E+00 | loss scale: 4096.0 | grad norm: 0.029 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.353 | TFLOPs: 72.78 | +[default7]: iteration 73/ 3814 | consumed samples: 37376 | consumed tokens: 76546048 | elapsed time per iteration (s): 5.31 | learning rate: 2.041E-04 | global batch size: 512 | lm loss: 9.827074E+00 | loss scale: 4096.0 | grad norm: 0.030 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.489 | TFLOPs: 72.88 | +[default7]: iteration 74/ 3814 | consumed samples: 37888 | consumed tokens: 77594624 | elapsed time per iteration (s): 5.30 | learning rate: 2.069E-04 | global batch size: 512 | lm loss: 9.803963E+00 | loss scale: 4096.0 | grad norm: 0.030 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.648 | TFLOPs: 73.00 | +[default7]: iteration 75/ 3814 | consumed samples: 38400 | consumed tokens: 78643200 | elapsed time per iteration (s): 5.32 | learning rate: 2.097E-04 | global batch size: 512 | lm loss: 9.770360E+00 | loss scale: 4096.0 | grad norm: 0.031 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.212 | TFLOPs: 72.67 | +[default7]: iteration 76/ 3814 | consumed samples: 38912 | consumed tokens: 79691776 | elapsed time per iteration (s): 5.33 | learning rate: 2.125E-04 | global batch size: 512 | lm loss: 9.746522E+00 | loss scale: 4096.0 | grad norm: 0.032 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.095 | TFLOPs: 72.58 | +[default7]: iteration 77/ 3814 | consumed samples: 39424 | consumed tokens: 80740352 | elapsed time per iteration (s): 5.32 | learning rate: 2.153E-04 | global batch size: 512 | lm loss: 9.718297E+00 | loss scale: 4096.0 | grad norm: 0.032 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.198 | TFLOPs: 72.66 | +[default7]: iteration 78/ 3814 | consumed samples: 39936 | consumed tokens: 81788928 | elapsed time per iteration (s): 5.32 | learning rate: 2.181E-04 | global batch size: 512 | lm loss: 9.684942E+00 | loss scale: 4096.0 | grad norm: 0.031 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.319 | TFLOPs: 72.75 | +[default7]: iteration 79/ 3814 | consumed samples: 40448 | consumed tokens: 82837504 | elapsed time per iteration (s): 5.31 | learning rate: 2.209E-04 | global batch size: 512 | lm loss: 9.655203E+00 | loss scale: 4096.0 | grad norm: 0.033 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.500 | TFLOPs: 72.89 | +[default7]: iteration 80/ 3814 | consumed samples: 40960 | consumed tokens: 83886080 | elapsed time per iteration (s): 5.31 | learning rate: 2.237E-04 | global batch size: 512 | lm loss: 9.627718E+00 | loss scale: 4096.0 | grad norm: 0.033 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.370 | TFLOPs: 72.79 | +[default7]: iteration 81/ 3814 | consumed samples: 41472 | consumed tokens: 84934656 | elapsed time per iteration (s): 5.32 | learning rate: 2.265E-04 | global batch size: 512 | lm loss: 9.592569E+00 | loss scale: 4096.0 | grad norm: 0.032 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.154 | TFLOPs: 72.63 | +[default7]: iteration 82/ 3814 | consumed samples: 41984 | consumed tokens: 85983232 | elapsed time per iteration (s): 5.32 | learning rate: 2.293E-04 | global batch size: 512 | lm loss: 9.569536E+00 | loss scale: 4096.0 | grad norm: 0.033 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.169 | TFLOPs: 72.64 | +[default7]: iteration 83/ 3814 | consumed samples: 42496 | consumed tokens: 87031808 | elapsed time per iteration (s): 5.30 | learning rate: 2.321E-04 | global batch size: 512 | lm loss: 9.530514E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.662 | TFLOPs: 73.01 | +[default7]: iteration 84/ 3814 | consumed samples: 43008 | consumed tokens: 88080384 | elapsed time per iteration (s): 5.31 | learning rate: 2.349E-04 | global batch size: 512 | lm loss: 9.492974E+00 | loss scale: 4096.0 | grad norm: 0.033 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.489 | TFLOPs: 72.88 | +[default7]: iteration 85/ 3814 | consumed samples: 43520 | consumed tokens: 89128960 | elapsed time per iteration (s): 5.30 | learning rate: 2.377E-04 | global batch size: 512 | lm loss: 9.479642E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.519 | TFLOPs: 72.90 | +[default7]: iteration 86/ 3814 | consumed samples: 44032 | consumed tokens: 90177536 | elapsed time per iteration (s): 5.32 | learning rate: 2.405E-04 | global batch size: 512 | lm loss: 9.435080E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.227 | TFLOPs: 72.68 | +[default7]: iteration 87/ 3814 | consumed samples: 44544 | consumed tokens: 91226112 | elapsed time per iteration (s): 5.31 | learning rate: 2.433E-04 | global batch size: 512 | lm loss: 9.411442E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.399 | TFLOPs: 72.81 | +[default7]: iteration 88/ 3814 | consumed samples: 45056 | consumed tokens: 92274688 | elapsed time per iteration (s): 5.31 | learning rate: 2.461E-04 | global batch size: 512 | lm loss: 9.374372E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.450 | TFLOPs: 72.85 | +[default7]: iteration 89/ 3814 | consumed samples: 45568 | consumed tokens: 93323264 | elapsed time per iteration (s): 5.32 | learning rate: 2.489E-04 | global batch size: 512 | lm loss: 9.335692E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.189 | TFLOPs: 72.65 | +[default7]: iteration 90/ 3814 | consumed samples: 46080 | consumed tokens: 94371840 | elapsed time per iteration (s): 5.30 | learning rate: 2.517E-04 | global batch size: 512 | lm loss: 9.303842E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.516 | TFLOPs: 72.90 | +[default7]: iteration 91/ 3814 | consumed samples: 46592 | consumed tokens: 95420416 | elapsed time per iteration (s): 5.30 | learning rate: 2.545E-04 | global batch size: 512 | lm loss: 9.279591E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.595 | TFLOPs: 72.96 | +[default7]: iteration 92/ 3814 | consumed samples: 47104 | consumed tokens: 96468992 | elapsed time per iteration (s): 5.31 | learning rate: 2.573E-04 | global batch size: 512 | lm loss: 9.242632E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.400 | TFLOPs: 72.81 | +[default7]: iteration 93/ 3814 | consumed samples: 47616 | consumed tokens: 97517568 | elapsed time per iteration (s): 5.33 | learning rate: 2.600E-04 | global batch size: 512 | lm loss: 9.206589E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.029 | TFLOPs: 72.53 | +[default7]: iteration 94/ 3814 | consumed samples: 48128 | consumed tokens: 98566144 | elapsed time per iteration (s): 5.32 | learning rate: 2.628E-04 | global batch size: 512 | lm loss: 9.173874E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.217 | TFLOPs: 72.67 | +[default7]: iteration 95/ 3814 | consumed samples: 48640 | consumed tokens: 99614720 | elapsed time per iteration (s): 5.31 | learning rate: 2.656E-04 | global batch size: 512 | lm loss: 9.149966E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.395 | TFLOPs: 72.81 | +[default7]: iteration 96/ 3814 | consumed samples: 49152 | consumed tokens: 100663296 | elapsed time per iteration (s): 5.32 | learning rate: 2.684E-04 | global batch size: 512 | lm loss: 9.104058E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.266 | TFLOPs: 72.71 | +[default7]: iteration 97/ 3814 | consumed samples: 49664 | consumed tokens: 101711872 | elapsed time per iteration (s): 5.30 | learning rate: 2.712E-04 | global batch size: 512 | lm loss: 9.077387E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.596 | TFLOPs: 72.96 | +[default7]: iteration 98/ 3814 | consumed samples: 50176 | consumed tokens: 102760448 | elapsed time per iteration (s): 5.31 | learning rate: 2.740E-04 | global batch size: 512 | lm loss: 9.040751E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.512 | TFLOPs: 72.90 | +[default7]: iteration 99/ 3814 | consumed samples: 50688 | consumed tokens: 103809024 | elapsed time per iteration (s): 5.31 | learning rate: 2.768E-04 | global batch size: 512 | lm loss: 9.009948E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.353 | TFLOPs: 72.78 | +[default7]: iteration 100/ 3814 | consumed samples: 51200 | consumed tokens: 104857600 | elapsed time per iteration (s): 5.32 | learning rate: 2.796E-04 | global batch size: 512 | lm loss: 8.981970E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.150 | TFLOPs: 72.62 | +[default7]: iteration 101/ 3814 | consumed samples: 51712 | consumed tokens: 105906176 | elapsed time per iteration (s): 5.31 | learning rate: 2.824E-04 | global batch size: 512 | lm loss: 8.948778E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.339 | TFLOPs: 72.77 | +[default7]: iteration 102/ 3814 | consumed samples: 52224 | consumed tokens: 106954752 | elapsed time per iteration (s): 5.30 | learning rate: 2.852E-04 | global batch size: 512 | lm loss: 8.910409E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.592 | TFLOPs: 72.96 | +[default7]: iteration 103/ 3814 | consumed samples: 52736 | consumed tokens: 108003328 | elapsed time per iteration (s): 5.32 | learning rate: 2.880E-04 | global batch size: 512 | lm loss: 8.861918E+00 | loss scale: 4096.0 | grad norm: 0.043 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.189 | TFLOPs: 72.65 | +[default7]: iteration 104/ 3814 | consumed samples: 53248 | consumed tokens: 109051904 | elapsed time per iteration (s): 5.32 | learning rate: 2.908E-04 | global batch size: 512 | lm loss: 8.841959E+00 | loss scale: 4096.0 | grad norm: 0.052 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.331 | TFLOPs: 72.76 | +[default7]: iteration 105/ 3814 | consumed samples: 53760 | consumed tokens: 110100480 | elapsed time per iteration (s): 5.30 | learning rate: 2.936E-04 | global batch size: 512 | lm loss: 8.806877E+00 | loss scale: 4096.0 | grad norm: 0.038 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.629 | TFLOPs: 72.98 | +[default7]: iteration 106/ 3814 | consumed samples: 54272 | consumed tokens: 111149056 | elapsed time per iteration (s): 5.31 | learning rate: 2.964E-04 | global batch size: 512 | lm loss: 8.772537E+00 | loss scale: 4096.0 | grad norm: 0.040 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.362 | TFLOPs: 72.78 | +[default7]: iteration 107/ 3814 | consumed samples: 54784 | consumed tokens: 112197632 | elapsed time per iteration (s): 5.31 | learning rate: 2.992E-04 | global batch size: 512 | lm loss: 8.737934E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.444 | TFLOPs: 72.84 | +[default7]: iteration 108/ 3814 | consumed samples: 55296 | consumed tokens: 113246208 | elapsed time per iteration (s): 5.29 | learning rate: 3.020E-04 | global batch size: 512 | lm loss: 8.713527E+00 | loss scale: 4096.0 | grad norm: 0.038 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.735 | TFLOPs: 73.06 | +[default7]: iteration 109/ 3814 | consumed samples: 55808 | consumed tokens: 114294784 | elapsed time per iteration (s): 5.29 | learning rate: 3.048E-04 | global batch size: 512 | lm loss: 8.657257E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.755 | TFLOPs: 73.08 | +[default7]: iteration 110/ 3814 | consumed samples: 56320 | consumed tokens: 115343360 | elapsed time per iteration (s): 5.30 | learning rate: 3.076E-04 | global batch size: 512 | lm loss: 8.655972E+00 | loss scale: 4096.0 | grad norm: 0.041 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.568 | TFLOPs: 72.94 | +[default7]: iteration 111/ 3814 | consumed samples: 56832 | consumed tokens: 116391936 | elapsed time per iteration (s): 5.31 | learning rate: 3.104E-04 | global batch size: 512 | lm loss: 8.602390E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.372 | TFLOPs: 72.79 | +[default7]: iteration 112/ 3814 | consumed samples: 57344 | consumed tokens: 117440512 | elapsed time per iteration (s): 5.33 | learning rate: 3.132E-04 | global batch size: 512 | lm loss: 8.573798E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.120 | TFLOPs: 72.60 | +[default7]: iteration 113/ 3814 | consumed samples: 57856 | consumed tokens: 118489088 | elapsed time per iteration (s): 5.30 | learning rate: 3.160E-04 | global batch size: 512 | lm loss: 8.535784E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.672 | TFLOPs: 73.02 | +[default7]: iteration 114/ 3814 | consumed samples: 58368 | consumed tokens: 119537664 | elapsed time per iteration (s): 5.31 | learning rate: 3.188E-04 | global batch size: 512 | lm loss: 8.507957E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.346 | TFLOPs: 72.77 | +[default7]: iteration 115/ 3814 | consumed samples: 58880 | consumed tokens: 120586240 | elapsed time per iteration (s): 5.29 | learning rate: 3.216E-04 | global batch size: 512 | lm loss: 8.483876E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.698 | TFLOPs: 73.04 | +[default7]: iteration 116/ 3814 | consumed samples: 59392 | consumed tokens: 121634816 | elapsed time per iteration (s): 5.30 | learning rate: 3.244E-04 | global batch size: 512 | lm loss: 8.464738E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.687 | TFLOPs: 73.03 | +[default7]: iteration 117/ 3814 | consumed samples: 59904 | consumed tokens: 122683392 | elapsed time per iteration (s): 5.33 | learning rate: 3.272E-04 | global batch size: 512 | lm loss: 8.401774E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.084 | TFLOPs: 72.57 | +[default7]: iteration 118/ 3814 | consumed samples: 60416 | consumed tokens: 123731968 | elapsed time per iteration (s): 5.33 | learning rate: 3.300E-04 | global batch size: 512 | lm loss: 8.374811E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.138 | TFLOPs: 72.61 | +[default7]: iteration 119/ 3814 | consumed samples: 60928 | consumed tokens: 124780544 | elapsed time per iteration (s): 5.35 | learning rate: 3.327E-04 | global batch size: 512 | lm loss: 8.338267E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.656 | TFLOPs: 72.25 | +[default7]: iteration 120/ 3814 | consumed samples: 61440 | consumed tokens: 125829120 | elapsed time per iteration (s): 5.31 | learning rate: 3.355E-04 | global batch size: 512 | lm loss: 8.309447E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.468 | TFLOPs: 72.86 | +[default7]: iteration 121/ 3814 | consumed samples: 61952 | consumed tokens: 126877696 | elapsed time per iteration (s): 5.33 | learning rate: 3.383E-04 | global batch size: 512 | lm loss: 8.270125E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.096 | TFLOPs: 72.58 | +[default7]: iteration 122/ 3814 | consumed samples: 62464 | consumed tokens: 127926272 | elapsed time per iteration (s): 5.31 | learning rate: 3.411E-04 | global batch size: 512 | lm loss: 8.247975E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.429 | TFLOPs: 72.83 | +[default7]: iteration 123/ 3814 | consumed samples: 62976 | consumed tokens: 128974848 | elapsed time per iteration (s): 5.30 | learning rate: 3.439E-04 | global batch size: 512 | lm loss: 8.217991E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.514 | TFLOPs: 72.90 | +[default7]: iteration 124/ 3814 | consumed samples: 63488 | consumed tokens: 130023424 | elapsed time per iteration (s): 5.31 | learning rate: 3.467E-04 | global batch size: 512 | lm loss: 8.190685E+00 | loss scale: 4096.0 | grad norm: 0.039 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.391 | TFLOPs: 72.80 | +[default7]: iteration 125/ 3814 | consumed samples: 64000 | consumed tokens: 131072000 | elapsed time per iteration (s): 5.31 | learning rate: 3.495E-04 | global batch size: 512 | lm loss: 8.165294E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.337 | TFLOPs: 72.76 | +[default7]: iteration 126/ 3814 | consumed samples: 64512 | consumed tokens: 132120576 | elapsed time per iteration (s): 5.32 | learning rate: 3.523E-04 | global batch size: 512 | lm loss: 8.113135E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.184 | TFLOPs: 72.65 | +[default7]: iteration 127/ 3814 | consumed samples: 65024 | consumed tokens: 133169152 | elapsed time per iteration (s): 5.30 | learning rate: 3.551E-04 | global batch size: 512 | lm loss: 8.105018E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.546 | TFLOPs: 72.92 | +[default7]: iteration 128/ 3814 | consumed samples: 65536 | consumed tokens: 134217728 | elapsed time per iteration (s): 5.29 | learning rate: 3.579E-04 | global batch size: 512 | lm loss: 8.086037E+00 | loss scale: 4096.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.710 | TFLOPs: 73.05 | +[default7]: iteration 129/ 3814 | consumed samples: 66048 | consumed tokens: 135266304 | elapsed time per iteration (s): 5.29 | learning rate: 3.607E-04 | global batch size: 512 | lm loss: 8.047988E+00 | loss scale: 4096.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.854 | TFLOPs: 73.16 | +[default7]: iteration 130/ 3814 | consumed samples: 66560 | consumed tokens: 136314880 | elapsed time per iteration (s): 5.33 | learning rate: 3.635E-04 | global batch size: 512 | lm loss: 8.004490E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.085 | TFLOPs: 72.57 | +[default7]: iteration 131/ 3814 | consumed samples: 67072 | consumed tokens: 137363456 | elapsed time per iteration (s): 5.36 | learning rate: 3.663E-04 | global batch size: 512 | lm loss: 7.994117E+00 | loss scale: 4096.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.587 | TFLOPs: 72.20 | +[default7]: iteration 132/ 3814 | consumed samples: 67584 | consumed tokens: 138412032 | elapsed time per iteration (s): 5.33 | learning rate: 3.691E-04 | global batch size: 512 | lm loss: 7.955549E+00 | loss scale: 4096.0 | grad norm: 0.036 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.064 | TFLOPs: 72.56 | +[default7]: iteration 133/ 3814 | consumed samples: 68096 | consumed tokens: 139460608 | elapsed time per iteration (s): 5.32 | learning rate: 3.719E-04 | global batch size: 512 | lm loss: 7.929621E+00 | loss scale: 4096.0 | grad norm: 0.035 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.183 | TFLOPs: 72.65 | +[default7]: iteration 134/ 3814 | consumed samples: 68608 | consumed tokens: 140509184 | elapsed time per iteration (s): 5.32 | learning rate: 3.747E-04 | global batch size: 512 | lm loss: 7.904654E+00 | loss scale: 4096.0 | grad norm: 0.032 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.252 | TFLOPs: 72.70 | +[default7]: iteration 135/ 3814 | consumed samples: 69120 | consumed tokens: 141557760 | elapsed time per iteration (s): 5.33 | learning rate: 3.775E-04 | global batch size: 512 | lm loss: 7.914517E+00 | loss scale: 4096.0 | grad norm: 0.042 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.000 | TFLOPs: 72.51 | +[default7]: iteration 136/ 3814 | consumed samples: 69632 | consumed tokens: 142606336 | elapsed time per iteration (s): 5.33 | learning rate: 3.803E-04 | global batch size: 512 | lm loss: 7.875209E+00 | loss scale: 4096.0 | grad norm: 0.038 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.097 | TFLOPs: 72.58 | +[default7]: iteration 137/ 3814 | consumed samples: 70144 | consumed tokens: 143654912 | elapsed time per iteration (s): 5.33 | learning rate: 3.831E-04 | global batch size: 512 | lm loss: 7.866493E+00 | loss scale: 4096.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.038 | TFLOPs: 72.54 | +[default7]: iteration 138/ 3814 | consumed samples: 70656 | consumed tokens: 144703488 | elapsed time per iteration (s): 5.34 | learning rate: 3.859E-04 | global batch size: 512 | lm loss: 7.804632E+00 | loss scale: 4096.0 | grad norm: 0.056 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.833 | TFLOPs: 72.38 | +[default7]: iteration 139/ 3814 | consumed samples: 71168 | consumed tokens: 145752064 | elapsed time per iteration (s): 5.34 | learning rate: 3.887E-04 | global batch size: 512 | lm loss: 7.812951E+00 | loss scale: 4096.0 | grad norm: 0.046 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.929 | TFLOPs: 72.46 | +[default7]: iteration 140/ 3814 | consumed samples: 71680 | consumed tokens: 146800640 | elapsed time per iteration (s): 5.33 | learning rate: 3.915E-04 | global batch size: 512 | lm loss: 7.771211E+00 | loss scale: 4096.0 | grad norm: 0.053 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.985 | TFLOPs: 72.50 | +[default7]: iteration 141/ 3814 | consumed samples: 72192 | consumed tokens: 147849216 | elapsed time per iteration (s): 5.35 | learning rate: 3.943E-04 | global batch size: 512 | lm loss: 7.728671E+00 | loss scale: 4096.0 | grad norm: 0.048 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.674 | TFLOPs: 72.26 | +[default7]: iteration 142/ 3814 | consumed samples: 72704 | consumed tokens: 148897792 | elapsed time per iteration (s): 5.34 | learning rate: 3.971E-04 | global batch size: 512 | lm loss: 7.702804E+00 | loss scale: 4096.0 | grad norm: 0.040 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.848 | TFLOPs: 72.39 | +[default7]: iteration 143/ 3814 | consumed samples: 73216 | consumed tokens: 149946368 | elapsed time per iteration (s): 5.35 | learning rate: 3.999E-04 | global batch size: 512 | lm loss: 7.756142E+00 | loss scale: 4096.0 | grad norm: 0.060 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.681 | TFLOPs: 72.27 | +[default7]: iteration 144/ 3814 | consumed samples: 73728 | consumed tokens: 150994944 | elapsed time per iteration (s): 5.34 | learning rate: 4.027E-04 | global batch size: 512 | lm loss: 7.700395E+00 | loss scale: 4096.0 | grad norm: 0.210 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.871 | TFLOPs: 72.41 | +[default7]: iteration 145/ 3814 | consumed samples: 74240 | consumed tokens: 152043520 | elapsed time per iteration (s): 5.32 | learning rate: 4.055E-04 | global batch size: 512 | lm loss: 7.691109E+00 | loss scale: 4096.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.180 | TFLOPs: 72.65 | +[default7]: iteration 146/ 3814 | consumed samples: 74752 | consumed tokens: 153092096 | elapsed time per iteration (s): 5.30 | learning rate: 4.082E-04 | global batch size: 512 | lm loss: 7.654244E+00 | loss scale: 4096.0 | grad norm: 0.034 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.545 | TFLOPs: 72.92 | +[default7]: iteration 147/ 3814 | consumed samples: 75264 | consumed tokens: 154140672 | elapsed time per iteration (s): 5.32 | learning rate: 4.110E-04 | global batch size: 512 | lm loss: 7.658628E+00 | loss scale: 4096.0 | grad norm: 0.052 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.267 | TFLOPs: 72.71 | +[default7]: iteration 148/ 3814 | consumed samples: 75776 | consumed tokens: 155189248 | elapsed time per iteration (s): 5.32 | learning rate: 4.138E-04 | global batch size: 512 | lm loss: 7.643138E+00 | loss scale: 4096.0 | grad norm: 0.063 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.214 | TFLOPs: 72.67 | +[default7]: iteration 149/ 3814 | consumed samples: 76288 | consumed tokens: 156237824 | elapsed time per iteration (s): 5.33 | learning rate: 4.166E-04 | global batch size: 512 | lm loss: 7.601076E+00 | loss scale: 4096.0 | grad norm: 0.054 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.101 | TFLOPs: 72.59 | +[default7]: iteration 150/ 3814 | consumed samples: 76800 | consumed tokens: 157286400 | elapsed time per iteration (s): 5.32 | learning rate: 4.194E-04 | global batch size: 512 | lm loss: 7.599413E+00 | loss scale: 4096.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.173 | TFLOPs: 72.64 | +[default7]: iteration 151/ 3814 | consumed samples: 77312 | consumed tokens: 158334976 | elapsed time per iteration (s): 5.33 | learning rate: 4.222E-04 | global batch size: 512 | lm loss: 7.584441E+00 | loss scale: 4096.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.037 | TFLOPs: 72.54 | +[default7]: iteration 152/ 3814 | consumed samples: 77824 | consumed tokens: 159383552 | elapsed time per iteration (s): 5.32 | learning rate: 4.250E-04 | global batch size: 512 | lm loss: 7.597820E+00 | loss scale: 4096.0 | grad norm: 0.152 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.249 | TFLOPs: 72.70 | +[default7]: iteration 153/ 3814 | consumed samples: 78336 | consumed tokens: 160432128 | elapsed time per iteration (s): 5.32 | learning rate: 4.278E-04 | global batch size: 512 | lm loss: 7.534708E+00 | loss scale: 4096.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.216 | TFLOPs: 72.67 | +[default7]: iteration 154/ 3814 | consumed samples: 78848 | consumed tokens: 161480704 | elapsed time per iteration (s): 5.33 | learning rate: 4.306E-04 | global batch size: 512 | lm loss: 7.542784E+00 | loss scale: 4096.0 | grad norm: 0.268 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.035 | TFLOPs: 72.54 | +[default7]: iteration 155/ 3814 | consumed samples: 79360 | consumed tokens: 162529280 | elapsed time per iteration (s): 5.32 | learning rate: 4.334E-04 | global batch size: 512 | lm loss: 7.523664E+00 | loss scale: 4096.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.237 | TFLOPs: 72.69 | +[default7]: iteration 156/ 3814 | consumed samples: 79872 | consumed tokens: 163577856 | elapsed time per iteration (s): 5.31 | learning rate: 4.362E-04 | global batch size: 512 | lm loss: 7.541519E+00 | loss scale: 4096.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.494 | TFLOPs: 72.88 | +[default7]: iteration 157/ 3814 | consumed samples: 80384 | consumed tokens: 164626432 | elapsed time per iteration (s): 5.30 | learning rate: 4.390E-04 | global batch size: 512 | lm loss: 7.491704E+00 | loss scale: 4096.0 | grad norm: 0.061 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.613 | TFLOPs: 72.97 | +[default7]: iteration 158/ 3814 | consumed samples: 80896 | consumed tokens: 165675008 | elapsed time per iteration (s): 5.31 | learning rate: 4.418E-04 | global batch size: 512 | lm loss: 7.469062E+00 | loss scale: 4096.0 | grad norm: 0.165 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.362 | TFLOPs: 72.78 | +[default7]: iteration 159/ 3814 | consumed samples: 81408 | consumed tokens: 166723584 | elapsed time per iteration (s): 5.32 | learning rate: 4.446E-04 | global batch size: 512 | lm loss: 7.511437E+00 | loss scale: 4096.0 | grad norm: 0.657 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.288 | TFLOPs: 72.73 | +[default7]: iteration 160/ 3814 | consumed samples: 81920 | consumed tokens: 167772160 | elapsed time per iteration (s): 5.32 | learning rate: 4.474E-04 | global batch size: 512 | lm loss: 7.493606E+00 | loss scale: 4096.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.292 | TFLOPs: 72.73 | +[default7]: iteration 161/ 3814 | consumed samples: 82432 | consumed tokens: 168820736 | elapsed time per iteration (s): 5.32 | learning rate: 4.502E-04 | global batch size: 512 | lm loss: 7.543811E+00 | loss scale: 4096.0 | grad norm: 0.403 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.245 | TFLOPs: 72.70 | +[default7]: iteration 162/ 3814 | consumed samples: 82944 | consumed tokens: 169869312 | elapsed time per iteration (s): 5.32 | learning rate: 4.530E-04 | global batch size: 512 | lm loss: 7.509077E+00 | loss scale: 4096.0 | grad norm: 0.272 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.179 | TFLOPs: 72.65 | +[default7]: iteration 163/ 3814 | consumed samples: 83456 | consumed tokens: 170917888 | elapsed time per iteration (s): 5.30 | learning rate: 4.558E-04 | global batch size: 512 | lm loss: 7.422496E+00 | loss scale: 4096.0 | grad norm: 0.177 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.603 | TFLOPs: 72.96 | +[default7]: iteration 164/ 3814 | consumed samples: 83968 | consumed tokens: 171966464 | elapsed time per iteration (s): 5.31 | learning rate: 4.586E-04 | global batch size: 512 | lm loss: 7.435603E+00 | loss scale: 4096.0 | grad norm: 0.281 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.399 | TFLOPs: 72.81 | +[default7]: iteration 165/ 3814 | consumed samples: 84480 | consumed tokens: 173015040 | elapsed time per iteration (s): 5.32 | learning rate: 4.614E-04 | global batch size: 512 | lm loss: 7.399710E+00 | loss scale: 4096.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.329 | TFLOPs: 72.76 | +[default7]: iteration 166/ 3814 | consumed samples: 84992 | consumed tokens: 174063616 | elapsed time per iteration (s): 5.32 | learning rate: 4.642E-04 | global batch size: 512 | lm loss: 7.384084E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.213 | TFLOPs: 72.67 | +[default7]: iteration 167/ 3814 | consumed samples: 85504 | consumed tokens: 175112192 | elapsed time per iteration (s): 5.32 | learning rate: 4.670E-04 | global batch size: 512 | lm loss: 7.389889E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.314 | TFLOPs: 72.75 | +[default7]: iteration 168/ 3814 | consumed samples: 86016 | consumed tokens: 176160768 | elapsed time per iteration (s): 5.33 | learning rate: 4.698E-04 | global batch size: 512 | lm loss: 7.349740E+00 | loss scale: 4096.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.113 | TFLOPs: 72.60 | +[default7]: iteration 169/ 3814 | consumed samples: 86528 | consumed tokens: 177209344 | elapsed time per iteration (s): 5.33 | learning rate: 4.726E-04 | global batch size: 512 | lm loss: 7.383105E+00 | loss scale: 4096.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.037 | TFLOPs: 72.54 | +[default7]: iteration 170/ 3814 | consumed samples: 87040 | consumed tokens: 178257920 | elapsed time per iteration (s): 5.33 | learning rate: 4.754E-04 | global batch size: 512 | lm loss: 7.308910E+00 | loss scale: 4096.0 | grad norm: 0.057 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.092 | TFLOPs: 72.58 | +[default7]: iteration 171/ 3814 | consumed samples: 87552 | consumed tokens: 179306496 | elapsed time per iteration (s): 5.30 | learning rate: 4.782E-04 | global batch size: 512 | lm loss: 7.344086E+00 | loss scale: 4096.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.631 | TFLOPs: 72.99 | +[default7]: iteration 172/ 3814 | consumed samples: 88064 | consumed tokens: 180355072 | elapsed time per iteration (s): 5.33 | learning rate: 4.809E-04 | global batch size: 512 | lm loss: 7.286698E+00 | loss scale: 4096.0 | grad norm: 0.062 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.060 | TFLOPs: 72.56 | +[default7]: iteration 173/ 3814 | consumed samples: 88576 | consumed tokens: 181403648 | elapsed time per iteration (s): 5.32 | learning rate: 4.837E-04 | global batch size: 512 | lm loss: 7.292053E+00 | loss scale: 4096.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.268 | TFLOPs: 72.71 | +[default7]: iteration 174/ 3814 | consumed samples: 89088 | consumed tokens: 182452224 | elapsed time per iteration (s): 5.31 | learning rate: 4.865E-04 | global batch size: 512 | lm loss: 7.303931E+00 | loss scale: 4096.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.375 | TFLOPs: 72.79 | +[default7]: iteration 175/ 3814 | consumed samples: 89600 | consumed tokens: 183500800 | elapsed time per iteration (s): 5.32 | learning rate: 4.893E-04 | global batch size: 512 | lm loss: 7.281268E+00 | loss scale: 4096.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.268 | TFLOPs: 72.71 | +[default7]: iteration 176/ 3814 | consumed samples: 90112 | consumed tokens: 184549376 | elapsed time per iteration (s): 5.30 | learning rate: 4.921E-04 | global batch size: 512 | lm loss: 7.285182E+00 | loss scale: 4096.0 | grad norm: 0.037 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.599 | TFLOPs: 72.96 | +[default7]: iteration 177/ 3814 | consumed samples: 90624 | consumed tokens: 185597952 | elapsed time per iteration (s): 5.33 | learning rate: 4.949E-04 | global batch size: 512 | lm loss: 7.263903E+00 | loss scale: 4096.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.128 | TFLOPs: 72.61 | +[default7]: iteration 178/ 3814 | consumed samples: 91136 | consumed tokens: 186646528 | elapsed time per iteration (s): 5.32 | learning rate: 4.977E-04 | global batch size: 512 | lm loss: 7.237145E+00 | loss scale: 4096.0 | grad norm: 0.041 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.294 | TFLOPs: 72.73 | +[default7]: iteration 179/ 3814 | consumed samples: 91648 | consumed tokens: 187695104 | elapsed time per iteration (s): 5.31 | learning rate: 5.005E-04 | global batch size: 512 | lm loss: 7.222631E+00 | loss scale: 4096.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.447 | TFLOPs: 72.85 | +[default7]: iteration 180/ 3814 | consumed samples: 92160 | consumed tokens: 188743680 | elapsed time per iteration (s): 5.32 | learning rate: 5.033E-04 | global batch size: 512 | lm loss: 7.193749E+00 | loss scale: 4096.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.254 | TFLOPs: 72.70 | +[default7]: iteration 181/ 3814 | consumed samples: 92672 | consumed tokens: 189792256 | elapsed time per iteration (s): 5.33 | learning rate: 5.061E-04 | global batch size: 512 | lm loss: 7.216022E+00 | loss scale: 4096.0 | grad norm: 0.188 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.004 | TFLOPs: 72.51 | +[default7]: iteration 182/ 3814 | consumed samples: 93184 | consumed tokens: 190840832 | elapsed time per iteration (s): 5.33 | learning rate: 5.089E-04 | global batch size: 512 | lm loss: 7.241582E+00 | loss scale: 4096.0 | grad norm: 0.179 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.005 | TFLOPs: 72.51 | +[default7]: iteration 183/ 3814 | consumed samples: 93696 | consumed tokens: 191889408 | elapsed time per iteration (s): 5.30 | learning rate: 5.117E-04 | global batch size: 512 | lm loss: 7.175785E+00 | loss scale: 4096.0 | grad norm: 0.059 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.562 | TFLOPs: 72.93 | +[default7]: iteration 184/ 3814 | consumed samples: 94208 | consumed tokens: 192937984 | elapsed time per iteration (s): 5.31 | learning rate: 5.145E-04 | global batch size: 512 | lm loss: 7.176373E+00 | loss scale: 4096.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.471 | TFLOPs: 72.87 | +[default7]: iteration 185/ 3814 | consumed samples: 94720 | consumed tokens: 193986560 | elapsed time per iteration (s): 5.31 | learning rate: 5.173E-04 | global batch size: 512 | lm loss: 7.186504E+00 | loss scale: 4096.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.420 | TFLOPs: 72.83 | +[default7]: iteration 186/ 3814 | consumed samples: 95232 | consumed tokens: 195035136 | elapsed time per iteration (s): 5.31 | learning rate: 5.201E-04 | global batch size: 512 | lm loss: 7.160543E+00 | loss scale: 4096.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.345 | TFLOPs: 72.77 | +[default7]: iteration 187/ 3814 | consumed samples: 95744 | consumed tokens: 196083712 | elapsed time per iteration (s): 5.31 | learning rate: 5.229E-04 | global batch size: 512 | lm loss: 7.134020E+00 | loss scale: 4096.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.408 | TFLOPs: 72.82 | +[default7]: iteration 188/ 3814 | consumed samples: 96256 | consumed tokens: 197132288 | elapsed time per iteration (s): 5.33 | learning rate: 5.257E-04 | global batch size: 512 | lm loss: 7.134689E+00 | loss scale: 4096.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.101 | TFLOPs: 72.59 | +[default7]: iteration 189/ 3814 | consumed samples: 96768 | consumed tokens: 198180864 | elapsed time per iteration (s): 5.32 | learning rate: 5.285E-04 | global batch size: 512 | lm loss: 7.125434E+00 | loss scale: 4096.0 | grad norm: 0.163 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.221 | TFLOPs: 72.68 | +[default7]: iteration 190/ 3814 | consumed samples: 97280 | consumed tokens: 199229440 | elapsed time per iteration (s): 5.31 | learning rate: 5.313E-04 | global batch size: 512 | lm loss: 7.146931E+00 | loss scale: 4096.0 | grad norm: 0.225 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.351 | TFLOPs: 72.78 | +[default7]: iteration 191/ 3814 | consumed samples: 97792 | consumed tokens: 200278016 | elapsed time per iteration (s): 5.30 | learning rate: 5.341E-04 | global batch size: 512 | lm loss: 7.115644E+00 | loss scale: 4096.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.587 | TFLOPs: 72.95 | +[default7]: iteration 192/ 3814 | consumed samples: 98304 | consumed tokens: 201326592 | elapsed time per iteration (s): 5.33 | learning rate: 5.369E-04 | global batch size: 512 | lm loss: 7.095028E+00 | loss scale: 4096.0 | grad norm: 0.190 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.117 | TFLOPs: 72.60 | +[default7]: iteration 193/ 3814 | consumed samples: 98816 | consumed tokens: 202375168 | elapsed time per iteration (s): 5.32 | learning rate: 5.397E-04 | global batch size: 512 | lm loss: 7.087056E+00 | loss scale: 4096.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.328 | TFLOPs: 72.76 | +[default7]: iteration 194/ 3814 | consumed samples: 99328 | consumed tokens: 203423744 | elapsed time per iteration (s): 5.31 | learning rate: 5.425E-04 | global batch size: 512 | lm loss: 7.062311E+00 | loss scale: 4096.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.445 | TFLOPs: 72.85 | +[default7]: iteration 195/ 3814 | consumed samples: 99840 | consumed tokens: 204472320 | elapsed time per iteration (s): 5.32 | learning rate: 5.453E-04 | global batch size: 512 | lm loss: 7.094954E+00 | loss scale: 4096.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.234 | TFLOPs: 72.69 | +[default7]: iteration 196/ 3814 | consumed samples: 100352 | consumed tokens: 205520896 | elapsed time per iteration (s): 5.32 | learning rate: 5.481E-04 | global batch size: 512 | lm loss: 7.078954E+00 | loss scale: 4096.0 | grad norm: 0.143 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.290 | TFLOPs: 72.73 | +[default7]: iteration 197/ 3814 | consumed samples: 100864 | consumed tokens: 206569472 | elapsed time per iteration (s): 5.30 | learning rate: 5.509E-04 | global batch size: 512 | lm loss: 7.041232E+00 | loss scale: 4096.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.562 | TFLOPs: 72.93 | +[default7]: iteration 198/ 3814 | consumed samples: 101376 | consumed tokens: 207618048 | elapsed time per iteration (s): 5.29 | learning rate: 5.536E-04 | global batch size: 512 | lm loss: 7.061014E+00 | loss scale: 4096.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.753 | TFLOPs: 73.08 | +[default7]: iteration 199/ 3814 | consumed samples: 101888 | consumed tokens: 208666624 | elapsed time per iteration (s): 5.32 | learning rate: 5.564E-04 | global batch size: 512 | lm loss: 7.033763E+00 | loss scale: 4096.0 | grad norm: 0.061 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.282 | TFLOPs: 72.72 | +[default7]: iteration 200/ 3814 | consumed samples: 102400 | consumed tokens: 209715200 | elapsed time per iteration (s): 5.34 | learning rate: 5.592E-04 | global batch size: 512 | lm loss: 6.998016E+00 | loss scale: 4096.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.815 | TFLOPs: 72.37 | +[default7]: iteration 201/ 3814 | consumed samples: 102912 | consumed tokens: 210763776 | elapsed time per iteration (s): 5.30 | learning rate: 5.620E-04 | global batch size: 512 | lm loss: 7.017907E+00 | loss scale: 4096.0 | grad norm: 0.052 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.557 | TFLOPs: 72.93 | +[default7]: iteration 202/ 3814 | consumed samples: 103424 | consumed tokens: 211812352 | elapsed time per iteration (s): 5.30 | learning rate: 5.648E-04 | global batch size: 512 | lm loss: 7.000439E+00 | loss scale: 4096.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.524 | TFLOPs: 72.91 | +[default7]: iteration 203/ 3814 | consumed samples: 103936 | consumed tokens: 212860928 | elapsed time per iteration (s): 5.31 | learning rate: 5.676E-04 | global batch size: 512 | lm loss: 6.979462E+00 | loss scale: 4096.0 | grad norm: 0.063 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.398 | TFLOPs: 72.81 | +[default7]: iteration 204/ 3814 | consumed samples: 104448 | consumed tokens: 213909504 | elapsed time per iteration (s): 5.33 | learning rate: 5.704E-04 | global batch size: 512 | lm loss: 6.959895E+00 | loss scale: 4096.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.119 | TFLOPs: 72.60 | +[default7]: iteration 205/ 3814 | consumed samples: 104960 | consumed tokens: 214958080 | elapsed time per iteration (s): 5.33 | learning rate: 5.732E-04 | global batch size: 512 | lm loss: 6.985322E+00 | loss scale: 4096.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.148 | TFLOPs: 72.62 | +[default7]: iteration 206/ 3814 | consumed samples: 105472 | consumed tokens: 216006656 | elapsed time per iteration (s): 5.31 | learning rate: 5.760E-04 | global batch size: 512 | lm loss: 6.964314E+00 | loss scale: 4096.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.495 | TFLOPs: 72.88 | +[default7]: iteration 207/ 3814 | consumed samples: 105984 | consumed tokens: 217055232 | elapsed time per iteration (s): 5.31 | learning rate: 5.788E-04 | global batch size: 512 | lm loss: 6.932876E+00 | loss scale: 4096.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.392 | TFLOPs: 72.81 | +[default7]: iteration 208/ 3814 | consumed samples: 106496 | consumed tokens: 218103808 | elapsed time per iteration (s): 5.30 | learning rate: 5.816E-04 | global batch size: 512 | lm loss: 6.945578E+00 | loss scale: 4096.0 | grad norm: 0.217 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.571 | TFLOPs: 72.94 | +[default7]: iteration 209/ 3814 | consumed samples: 107008 | consumed tokens: 219152384 | elapsed time per iteration (s): 5.32 | learning rate: 5.844E-04 | global batch size: 512 | lm loss: 6.930402E+00 | loss scale: 4096.0 | grad norm: 0.232 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.299 | TFLOPs: 72.74 | +[default7]: iteration 210/ 3814 | consumed samples: 107520 | consumed tokens: 220200960 | elapsed time per iteration (s): 5.31 | learning rate: 5.872E-04 | global batch size: 512 | lm loss: 6.923115E+00 | loss scale: 4096.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.465 | TFLOPs: 72.86 | +[default7]: iteration 211/ 3814 | consumed samples: 108032 | consumed tokens: 221249536 | elapsed time per iteration (s): 5.32 | learning rate: 5.900E-04 | global batch size: 512 | lm loss: 6.871114E+00 | loss scale: 4096.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.292 | TFLOPs: 72.73 | +[default7]: iteration 212/ 3814 | consumed samples: 108544 | consumed tokens: 222298112 | elapsed time per iteration (s): 5.33 | learning rate: 5.928E-04 | global batch size: 512 | lm loss: 6.915003E+00 | loss scale: 4096.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.111 | TFLOPs: 72.59 | +[default7]: iteration 213/ 3814 | consumed samples: 109056 | consumed tokens: 223346688 | elapsed time per iteration (s): 5.33 | learning rate: 5.956E-04 | global batch size: 512 | lm loss: 6.870727E+00 | loss scale: 4096.0 | grad norm: 0.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.085 | TFLOPs: 72.57 | +[default7]: iteration 214/ 3814 | consumed samples: 109568 | consumed tokens: 224395264 | elapsed time per iteration (s): 5.33 | learning rate: 5.984E-04 | global batch size: 512 | lm loss: 6.866611E+00 | loss scale: 4096.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.067 | TFLOPs: 72.56 | +[default7]: iteration 215/ 3814 | consumed samples: 110080 | consumed tokens: 225443840 | elapsed time per iteration (s): 5.31 | learning rate: 6.012E-04 | global batch size: 512 | lm loss: 6.875516E+00 | loss scale: 4096.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.379 | TFLOPs: 72.80 | +[default7]: iteration 216/ 3814 | consumed samples: 110592 | consumed tokens: 226492416 | elapsed time per iteration (s): 5.33 | learning rate: 6.040E-04 | global batch size: 512 | lm loss: 6.889411E+00 | loss scale: 4096.0 | grad norm: 0.202 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.128 | TFLOPs: 72.61 | +[default7]: iteration 217/ 3814 | consumed samples: 111104 | consumed tokens: 227540992 | elapsed time per iteration (s): 5.34 | learning rate: 6.068E-04 | global batch size: 512 | lm loss: 6.864443E+00 | loss scale: 4096.0 | grad norm: 0.200 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.800 | TFLOPs: 72.36 | +[default7]: iteration 218/ 3814 | consumed samples: 111616 | consumed tokens: 228589568 | elapsed time per iteration (s): 5.33 | learning rate: 6.096E-04 | global batch size: 512 | lm loss: 6.835349E+00 | loss scale: 4096.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.102 | TFLOPs: 72.59 | +[default7]: iteration 219/ 3814 | consumed samples: 112128 | consumed tokens: 229638144 | elapsed time per iteration (s): 5.34 | learning rate: 6.124E-04 | global batch size: 512 | lm loss: 6.822618E+00 | loss scale: 4096.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.883 | TFLOPs: 72.42 | +[default7]: iteration 220/ 3814 | consumed samples: 112640 | consumed tokens: 230686720 | elapsed time per iteration (s): 5.33 | learning rate: 6.152E-04 | global batch size: 512 | lm loss: 6.819791E+00 | loss scale: 4096.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.082 | TFLOPs: 72.57 | +[default7]: iteration 221/ 3814 | consumed samples: 113152 | consumed tokens: 231735296 | elapsed time per iteration (s): 5.36 | learning rate: 6.180E-04 | global batch size: 512 | lm loss: 6.848757E+00 | loss scale: 4096.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.471 | TFLOPs: 72.11 | +[default7]: iteration 222/ 3814 | consumed samples: 113664 | consumed tokens: 232783872 | elapsed time per iteration (s): 5.32 | learning rate: 6.208E-04 | global batch size: 512 | lm loss: 6.812945E+00 | loss scale: 4096.0 | grad norm: 0.163 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.329 | TFLOPs: 72.76 | +[default7]: iteration 223/ 3814 | consumed samples: 114176 | consumed tokens: 233832448 | elapsed time per iteration (s): 5.31 | learning rate: 6.236E-04 | global batch size: 512 | lm loss: 6.794768E+00 | loss scale: 4096.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.369 | TFLOPs: 72.79 | +[default7]: iteration 224/ 3814 | consumed samples: 114688 | consumed tokens: 234881024 | elapsed time per iteration (s): 5.32 | learning rate: 6.264E-04 | global batch size: 512 | lm loss: 6.785290E+00 | loss scale: 4096.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.310 | TFLOPs: 72.74 | +[default7]: iteration 225/ 3814 | consumed samples: 115200 | consumed tokens: 235929600 | elapsed time per iteration (s): 5.33 | learning rate: 6.291E-04 | global batch size: 512 | lm loss: 6.772423E+00 | loss scale: 4096.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.121 | TFLOPs: 72.60 | +[default7]: iteration 226/ 3814 | consumed samples: 115712 | consumed tokens: 236978176 | elapsed time per iteration (s): 5.31 | learning rate: 6.319E-04 | global batch size: 512 | lm loss: 6.772120E+00 | loss scale: 4096.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.351 | TFLOPs: 72.77 | +[default7]: iteration 227/ 3814 | consumed samples: 116224 | consumed tokens: 238026752 | elapsed time per iteration (s): 5.33 | learning rate: 6.347E-04 | global batch size: 512 | lm loss: 6.756733E+00 | loss scale: 4096.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.115 | TFLOPs: 72.60 | +[default7]: iteration 228/ 3814 | consumed samples: 116736 | consumed tokens: 239075328 | elapsed time per iteration (s): 5.33 | learning rate: 6.375E-04 | global batch size: 512 | lm loss: 6.736584E+00 | loss scale: 4096.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.991 | TFLOPs: 72.50 | +[default7]: iteration 229/ 3814 | consumed samples: 117248 | consumed tokens: 240123904 | elapsed time per iteration (s): 5.34 | learning rate: 6.403E-04 | global batch size: 512 | lm loss: 6.731544E+00 | loss scale: 4096.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.943 | TFLOPs: 72.47 | +[default7]: iteration 230/ 3814 | consumed samples: 117760 | consumed tokens: 241172480 | elapsed time per iteration (s): 5.32 | learning rate: 6.431E-04 | global batch size: 512 | lm loss: 6.710187E+00 | loss scale: 4096.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.249 | TFLOPs: 72.70 | +[default7]: iteration 231/ 3814 | consumed samples: 118272 | consumed tokens: 242221056 | elapsed time per iteration (s): 5.33 | learning rate: 6.459E-04 | global batch size: 512 | lm loss: 6.711408E+00 | loss scale: 4096.0 | grad norm: 0.148 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.065 | TFLOPs: 72.56 | +[default7]: iteration 232/ 3814 | consumed samples: 118784 | consumed tokens: 243269632 | elapsed time per iteration (s): 5.34 | learning rate: 6.487E-04 | global batch size: 512 | lm loss: 6.714088E+00 | loss scale: 4096.0 | grad norm: 0.177 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.887 | TFLOPs: 72.42 | +[default7]: iteration 233/ 3814 | consumed samples: 119296 | consumed tokens: 244318208 | elapsed time per iteration (s): 5.34 | learning rate: 6.515E-04 | global batch size: 512 | lm loss: 6.692508E+00 | loss scale: 4096.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.862 | TFLOPs: 72.41 | +[default7]: iteration 234/ 3814 | consumed samples: 119808 | consumed tokens: 245366784 | elapsed time per iteration (s): 5.32 | learning rate: 6.543E-04 | global batch size: 512 | lm loss: 6.667068E+00 | loss scale: 4096.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.180 | TFLOPs: 72.65 | +[default7]: iteration 235/ 3814 | consumed samples: 120320 | consumed tokens: 246415360 | elapsed time per iteration (s): 5.32 | learning rate: 6.571E-04 | global batch size: 512 | lm loss: 6.688784E+00 | loss scale: 4096.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.249 | TFLOPs: 72.70 | +[default7]: iteration 236/ 3814 | consumed samples: 120832 | consumed tokens: 247463936 | elapsed time per iteration (s): 5.35 | learning rate: 6.599E-04 | global batch size: 512 | lm loss: 6.663113E+00 | loss scale: 4096.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.614 | TFLOPs: 72.22 | +[default7]: iteration 237/ 3814 | consumed samples: 121344 | consumed tokens: 248512512 | elapsed time per iteration (s): 5.36 | learning rate: 6.627E-04 | global batch size: 512 | lm loss: 6.682841E+00 | loss scale: 4096.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.528 | TFLOPs: 72.15 | +[default7]: iteration 238/ 3814 | consumed samples: 121856 | consumed tokens: 249561088 | elapsed time per iteration (s): 5.33 | learning rate: 6.655E-04 | global batch size: 512 | lm loss: 6.653779E+00 | loss scale: 4096.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.000 | TFLOPs: 72.51 | +[default7]: iteration 239/ 3814 | consumed samples: 122368 | consumed tokens: 250609664 | elapsed time per iteration (s): 5.33 | learning rate: 6.683E-04 | global batch size: 512 | lm loss: 6.663728E+00 | loss scale: 4096.0 | grad norm: 0.216 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.099 | TFLOPs: 72.58 | +[default7]: iteration 240/ 3814 | consumed samples: 122880 | consumed tokens: 251658240 | elapsed time per iteration (s): 5.34 | learning rate: 6.711E-04 | global batch size: 512 | lm loss: 6.656242E+00 | loss scale: 4096.0 | grad norm: 0.169 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.962 | TFLOPs: 72.48 | +[default7]: iteration 241/ 3814 | consumed samples: 123392 | consumed tokens: 252706816 | elapsed time per iteration (s): 5.32 | learning rate: 6.739E-04 | global batch size: 512 | lm loss: 6.651799E+00 | loss scale: 4096.0 | grad norm: 0.196 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.227 | TFLOPs: 72.68 | +[default7]: iteration 242/ 3814 | consumed samples: 123904 | consumed tokens: 253755392 | elapsed time per iteration (s): 5.33 | learning rate: 6.767E-04 | global batch size: 512 | lm loss: 6.651060E+00 | loss scale: 4096.0 | grad norm: 0.158 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.012 | TFLOPs: 72.52 | +[default7]: iteration 243/ 3814 | consumed samples: 124416 | consumed tokens: 254803968 | elapsed time per iteration (s): 5.35 | learning rate: 6.795E-04 | global batch size: 512 | lm loss: 6.593741E+00 | loss scale: 4096.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.778 | TFLOPs: 72.34 | +[default7]: iteration 244/ 3814 | consumed samples: 124928 | consumed tokens: 255852544 | elapsed time per iteration (s): 5.34 | learning rate: 6.823E-04 | global batch size: 512 | lm loss: 6.633750E+00 | loss scale: 4096.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.965 | TFLOPs: 72.48 | +[default7]: iteration 245/ 3814 | consumed samples: 125440 | consumed tokens: 256901120 | elapsed time per iteration (s): 5.32 | learning rate: 6.851E-04 | global batch size: 512 | lm loss: 6.614361E+00 | loss scale: 4096.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.290 | TFLOPs: 72.73 | +[default7]: iteration 246/ 3814 | consumed samples: 125952 | consumed tokens: 257949696 | elapsed time per iteration (s): 5.33 | learning rate: 6.879E-04 | global batch size: 512 | lm loss: 6.618662E+00 | loss scale: 4096.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.072 | TFLOPs: 72.56 | +[default7]: iteration 247/ 3814 | consumed samples: 126464 | consumed tokens: 258998272 | elapsed time per iteration (s): 5.32 | learning rate: 6.907E-04 | global batch size: 512 | lm loss: 6.565890E+00 | loss scale: 4096.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.212 | TFLOPs: 72.67 | +[default7]: iteration 248/ 3814 | consumed samples: 126976 | consumed tokens: 260046848 | elapsed time per iteration (s): 5.35 | learning rate: 6.935E-04 | global batch size: 512 | lm loss: 6.572883E+00 | loss scale: 4096.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.636 | TFLOPs: 72.24 | +[default7]: iteration 249/ 3814 | consumed samples: 127488 | consumed tokens: 261095424 | elapsed time per iteration (s): 5.34 | learning rate: 6.963E-04 | global batch size: 512 | lm loss: 6.595792E+00 | loss scale: 4096.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.862 | TFLOPs: 72.41 | +[default7]: iteration 250/ 3814 | consumed samples: 128000 | consumed tokens: 262144000 | elapsed time per iteration (s): 5.34 | learning rate: 6.991E-04 | global batch size: 512 | lm loss: 6.586693E+00 | loss scale: 4096.0 | grad norm: 0.184 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.914 | TFLOPs: 72.44 | +[default1]:[2023-02-16 06:42:33,745] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_01-model_01-model_states.pt... +[default0]:saving checkpoint at iteration 250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 06:42:33,700] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step250 is begin to save! +[default0]:[2023-02-16 06:42:33,745] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:33,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:33,939] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_04-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:34,009] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,010] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:33,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:33,985] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,070] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:34,154] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,154] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,108] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,108] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_05-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:34,179] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_05-model_00-model_states.pt. +[default1]:[2023-02-16 06:42:34,237] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,237] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_08-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,180] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:34,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,247] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,314] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,307] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,307] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,372] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:34,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,451] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,386] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,387] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_09-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:34,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,448] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,511] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,512] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,528] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,528] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,590] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,590] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_13-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:34,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,652] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,588] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,588] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_12-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:34,669] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,670] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_13-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,734] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,732] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,797] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,812] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:34,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,872] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:34,874] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,875] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:34,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:34,938] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:34,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:34,952] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,017] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_19-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,012] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,079] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_20-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:35,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_21-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,095] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:35,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,233] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,234] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,251] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,252] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,313] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,316] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:35,378] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,379] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:35,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,455] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,459] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,522] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,596] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,596] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,610] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,611] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_26-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:35,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,675] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_28-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:35,737] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,759] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,821] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:35,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,900] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:35,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:35,962] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:35,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,887] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:35,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:35,969] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:36,044] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:36,044] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:36,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,034] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:36,113] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:36,113] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 06:42:36,114] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,115] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_33-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:36,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,181] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:36,197] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:36,197] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:36,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:36,260] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 06:42:36,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 06:42:36,262] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_01_model_states.pt +[default1]:[2023-02-16 06:42:36,262] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_01_model_states.pt... +[default0]:[2023-02-16 06:42:36,263] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_35-model_00-model_states.pt... +[default1]:[2023-02-16 06:42:36,268] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_01_model_states.pt. +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 06:42:36,324] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,324] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 06:42:36,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 06:42:36,326] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_00_model_states.pt +[default0]:[2023-02-16 06:42:36,326] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 06:42:36,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 06:42:36,366] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 06:42:36,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:36,821] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:36,822] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:36,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:36,904] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:36,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:36,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:36,931] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:36,931] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:36,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:36,968] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:36,968] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:36,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:36,973] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:36,973] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,009] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,010] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,010] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:36,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:36,939] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:36,939] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:36,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:36,950] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:36,950] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:36,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:36,987] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:36,987] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:36,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:36,973] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:36,973] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,005] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,005] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:36,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:36,949] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:36,949] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:37,048] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:37,048] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:37,048] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,024] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,024] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,065] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,065] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,065] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:36,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:36,993] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:36,993] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:36,998] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:36,998] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:36,998] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,012] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,039] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,039] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,049] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,049] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:37,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:37,041] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:37,041] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:37,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:37,006] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:37,006] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,087] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,087] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,088] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,033] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,033] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,086] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,086] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,046] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,046] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:37,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:37,037] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:37,037] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,065] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,065] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,065] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:37,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:37,086] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:37,086] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,041] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,041] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,086] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,086] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,086] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:37,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:37,058] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:37,058] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,058] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,058] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,072] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,072] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:37,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:37,130] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:37,130] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default5]:[2023-02-16 06:42:37,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:37,094] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:37,094] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:37,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:37,152] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:37,152] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:37,113] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:37,113] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:37,113] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,110] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,110] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,119] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,119] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,119] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,094] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,094] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:37,101] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:37,101] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:37,101] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,173] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,173] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,173] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,136] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,136] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,136] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:37,112] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:37,112] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:37,112] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,150] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,150] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,171] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,171] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,204] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,204] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default3]:[2023-02-16 06:42:37,136] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 06:42:37,136] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 06:42:37,136] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,209] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,210] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,210] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,151] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,151] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,200] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,200] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,142] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,142] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,142] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,169] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,169] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,169] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default6]:[2023-02-16 06:42:37,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 06:42:37,225] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 06:42:37,225] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,206] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,207] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,207] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default4]:[2023-02-16 06:42:37,214] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,214] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,214] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,213] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,213] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,213] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default1]:[2023-02-16 06:42:37,177] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 06:42:37,178] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 06:42:37,178] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default2]:[2023-02-16 06:42:37,228] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 06:42:37,228] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 06:42:37,228] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]: successfully saved checkpoint at iteration 250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default5]:[2023-02-16 06:42:37,271] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 06:42:37,271] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 06:42:37,271] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:[2023-02-16 06:42:37,273] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 06:42:37,273] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 06:42:37,273] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]:time (ms) | save-checkpoint: 3592.30 +[default4]:[2023-02-16 06:42:37,226] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 06:42:37,226] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 06:42:37,227] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default0]:[2023-02-16 06:42:37,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 06:42:37,276] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step250/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 06:42:37,276] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step250 is ready now! +[default7]: iteration 251/ 3814 | consumed samples: 128512 | consumed tokens: 263192576 | elapsed time per iteration (s): 8.93 | learning rate: 7.018E-04 | global batch size: 512 | lm loss: 6.599677E+00 | loss scale: 4096.0 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 57.365 | TFLOPs: 43.33 | +[default7]: iteration 252/ 3814 | consumed samples: 129024 | consumed tokens: 264241152 | elapsed time per iteration (s): 5.32 | learning rate: 7.046E-04 | global batch size: 512 | lm loss: 6.591994E+00 | loss scale: 4096.0 | grad norm: 0.284 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.174 | TFLOPs: 72.64 | +[default7]: iteration 253/ 3814 | consumed samples: 129536 | consumed tokens: 265289728 | elapsed time per iteration (s): 5.35 | learning rate: 7.074E-04 | global batch size: 512 | lm loss: 6.581171E+00 | loss scale: 4096.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.693 | TFLOPs: 72.28 | +[default7]: iteration 254/ 3814 | consumed samples: 130048 | consumed tokens: 266338304 | elapsed time per iteration (s): 5.34 | learning rate: 7.102E-04 | global batch size: 512 | lm loss: 6.542196E+00 | loss scale: 4096.0 | grad norm: 0.190 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.897 | TFLOPs: 72.43 | +[default7]: iteration 255/ 3814 | consumed samples: 130560 | consumed tokens: 267386880 | elapsed time per iteration (s): 5.33 | learning rate: 7.130E-04 | global batch size: 512 | lm loss: 6.552608E+00 | loss scale: 4096.0 | grad norm: 0.212 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.003 | TFLOPs: 72.51 | +[default7]: iteration 256/ 3814 | consumed samples: 131072 | consumed tokens: 268435456 | elapsed time per iteration (s): 5.33 | learning rate: 7.158E-04 | global batch size: 512 | lm loss: 6.544002E+00 | loss scale: 4096.0 | grad norm: 0.204 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.019 | TFLOPs: 72.52 | +[default7]: iteration 257/ 3814 | consumed samples: 131584 | consumed tokens: 269484032 | elapsed time per iteration (s): 5.33 | learning rate: 7.186E-04 | global batch size: 512 | lm loss: 6.559335E+00 | loss scale: 4096.0 | grad norm: 0.227 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.012 | TFLOPs: 72.52 | +[default7]: iteration 258/ 3814 | consumed samples: 132096 | consumed tokens: 270532608 | elapsed time per iteration (s): 5.36 | learning rate: 7.214E-04 | global batch size: 512 | lm loss: 6.558733E+00 | loss scale: 4096.0 | grad norm: 0.268 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.538 | TFLOPs: 72.16 | +[default7]: iteration 259/ 3814 | consumed samples: 132608 | consumed tokens: 271581184 | elapsed time per iteration (s): 5.35 | learning rate: 7.242E-04 | global batch size: 512 | lm loss: 6.512566E+00 | loss scale: 4096.0 | grad norm: 0.175 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.702 | TFLOPs: 72.28 | +[default7]: iteration 260/ 3814 | consumed samples: 133120 | consumed tokens: 272629760 | elapsed time per iteration (s): 5.35 | learning rate: 7.270E-04 | global batch size: 512 | lm loss: 6.501917E+00 | loss scale: 4096.0 | grad norm: 0.154 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.763 | TFLOPs: 72.33 | +[default7]: iteration 261/ 3814 | consumed samples: 133632 | consumed tokens: 273678336 | elapsed time per iteration (s): 5.35 | learning rate: 7.298E-04 | global batch size: 512 | lm loss: 6.511424E+00 | loss scale: 4096.0 | grad norm: 0.143 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.788 | TFLOPs: 72.35 | +[default7]: iteration 262/ 3814 | consumed samples: 134144 | consumed tokens: 274726912 | elapsed time per iteration (s): 5.33 | learning rate: 7.326E-04 | global batch size: 512 | lm loss: 6.488906E+00 | loss scale: 4096.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.033 | TFLOPs: 72.53 | +[default7]: iteration 263/ 3814 | consumed samples: 134656 | consumed tokens: 275775488 | elapsed time per iteration (s): 5.34 | learning rate: 7.354E-04 | global batch size: 512 | lm loss: 6.488328E+00 | loss scale: 4096.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.838 | TFLOPs: 72.39 | +[default7]: iteration 264/ 3814 | consumed samples: 135168 | consumed tokens: 276824064 | elapsed time per iteration (s): 5.35 | learning rate: 7.382E-04 | global batch size: 512 | lm loss: 6.454405E+00 | loss scale: 4096.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.675 | TFLOPs: 72.26 | +[default7]: iteration 265/ 3814 | consumed samples: 135680 | consumed tokens: 277872640 | elapsed time per iteration (s): 5.34 | learning rate: 7.410E-04 | global batch size: 512 | lm loss: 6.503987E+00 | loss scale: 4096.0 | grad norm: 0.156 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.924 | TFLOPs: 72.45 | +[default7]: iteration 266/ 3814 | consumed samples: 136192 | consumed tokens: 278921216 | elapsed time per iteration (s): 5.34 | learning rate: 7.438E-04 | global batch size: 512 | lm loss: 6.517190E+00 | loss scale: 4096.0 | grad norm: 0.184 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.948 | TFLOPs: 72.47 | +[default7]: iteration 267/ 3814 | consumed samples: 136704 | consumed tokens: 279969792 | elapsed time per iteration (s): 5.34 | learning rate: 7.466E-04 | global batch size: 512 | lm loss: 6.474828E+00 | loss scale: 4096.0 | grad norm: 0.152 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.941 | TFLOPs: 72.47 | +[default7]: iteration 268/ 3814 | consumed samples: 137216 | consumed tokens: 281018368 | elapsed time per iteration (s): 5.34 | learning rate: 7.494E-04 | global batch size: 512 | lm loss: 6.486125E+00 | loss scale: 4096.0 | grad norm: 0.170 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.794 | TFLOPs: 72.35 | +[default7]: iteration 269/ 3814 | consumed samples: 137728 | consumed tokens: 282066944 | elapsed time per iteration (s): 5.35 | learning rate: 7.522E-04 | global batch size: 512 | lm loss: 6.483952E+00 | loss scale: 4096.0 | grad norm: 0.198 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.763 | TFLOPs: 72.33 | +[default7]: iteration 270/ 3814 | consumed samples: 138240 | consumed tokens: 283115520 | elapsed time per iteration (s): 5.35 | learning rate: 7.550E-04 | global batch size: 512 | lm loss: 6.441314E+00 | loss scale: 4096.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.766 | TFLOPs: 72.33 | +[default7]: iteration 271/ 3814 | consumed samples: 138752 | consumed tokens: 284164096 | elapsed time per iteration (s): 5.34 | learning rate: 7.578E-04 | global batch size: 512 | lm loss: 6.459949E+00 | loss scale: 4096.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.872 | TFLOPs: 72.41 | +[default7]: iteration 272/ 3814 | consumed samples: 139264 | consumed tokens: 285212672 | elapsed time per iteration (s): 5.35 | learning rate: 7.606E-04 | global batch size: 512 | lm loss: 6.445519E+00 | loss scale: 4096.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.760 | TFLOPs: 72.33 | +[default7]: iteration 273/ 3814 | consumed samples: 139776 | consumed tokens: 286261248 | elapsed time per iteration (s): 5.35 | learning rate: 7.634E-04 | global batch size: 512 | lm loss: 6.406525E+00 | loss scale: 4096.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.635 | TFLOPs: 72.23 | +[default7]: iteration 274/ 3814 | consumed samples: 140288 | consumed tokens: 287309824 | elapsed time per iteration (s): 5.35 | learning rate: 7.662E-04 | global batch size: 512 | lm loss: 6.444201E+00 | loss scale: 4096.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.675 | TFLOPs: 72.26 | +[default7]: iteration 275/ 3814 | consumed samples: 140800 | consumed tokens: 288358400 | elapsed time per iteration (s): 5.33 | learning rate: 7.690E-04 | global batch size: 512 | lm loss: 6.418411E+00 | loss scale: 4096.0 | grad norm: 0.152 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.066 | TFLOPs: 72.56 | +[default7]: iteration 276/ 3814 | consumed samples: 141312 | consumed tokens: 289406976 | elapsed time per iteration (s): 5.34 | learning rate: 7.718E-04 | global batch size: 512 | lm loss: 6.417141E+00 | loss scale: 4096.0 | grad norm: 0.298 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.968 | TFLOPs: 72.49 | +[default7]: iteration 277/ 3814 | consumed samples: 141824 | consumed tokens: 290455552 | elapsed time per iteration (s): 5.36 | learning rate: 7.746E-04 | global batch size: 512 | lm loss: 6.421048E+00 | loss scale: 4096.0 | grad norm: 0.205 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.441 | TFLOPs: 72.09 | +[default7]: iteration 278/ 3814 | consumed samples: 142336 | consumed tokens: 291504128 | elapsed time per iteration (s): 5.34 | learning rate: 7.773E-04 | global batch size: 512 | lm loss: 6.418058E+00 | loss scale: 4096.0 | grad norm: 0.154 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.873 | TFLOPs: 72.41 | +[default7]: iteration 279/ 3814 | consumed samples: 142848 | consumed tokens: 292552704 | elapsed time per iteration (s): 5.33 | learning rate: 7.801E-04 | global batch size: 512 | lm loss: 6.406618E+00 | loss scale: 4096.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.984 | TFLOPs: 72.50 | +[default7]: iteration 280/ 3814 | consumed samples: 143360 | consumed tokens: 293601280 | elapsed time per iteration (s): 5.33 | learning rate: 7.829E-04 | global batch size: 512 | lm loss: 6.384571E+00 | loss scale: 4096.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.082 | TFLOPs: 72.57 | +[default7]: iteration 281/ 3814 | consumed samples: 143872 | consumed tokens: 294649856 | elapsed time per iteration (s): 5.34 | learning rate: 7.857E-04 | global batch size: 512 | lm loss: 6.433259E+00 | loss scale: 4096.0 | grad norm: 0.199 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.867 | TFLOPs: 72.41 | +[default7]: iteration 282/ 3814 | consumed samples: 144384 | consumed tokens: 295698432 | elapsed time per iteration (s): 5.36 | learning rate: 7.885E-04 | global batch size: 512 | lm loss: 6.415375E+00 | loss scale: 4096.0 | grad norm: 0.314 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.583 | TFLOPs: 72.19 | +[default7]: iteration 283/ 3814 | consumed samples: 144896 | consumed tokens: 296747008 | elapsed time per iteration (s): 5.35 | learning rate: 7.913E-04 | global batch size: 512 | lm loss: 6.423252E+00 | loss scale: 4096.0 | grad norm: 0.305 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.787 | TFLOPs: 72.35 | +[default7]: iteration 284/ 3814 | consumed samples: 145408 | consumed tokens: 297795584 | elapsed time per iteration (s): 5.33 | learning rate: 7.941E-04 | global batch size: 512 | lm loss: 6.358175E+00 | loss scale: 4096.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.034 | TFLOPs: 72.54 | +[default7]: iteration 285/ 3814 | consumed samples: 145920 | consumed tokens: 298844160 | elapsed time per iteration (s): 5.33 | learning rate: 7.969E-04 | global batch size: 512 | lm loss: 6.395652E+00 | loss scale: 4096.0 | grad norm: 0.204 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.056 | TFLOPs: 72.55 | +[default7]: iteration 286/ 3814 | consumed samples: 146432 | consumed tokens: 299892736 | elapsed time per iteration (s): 5.34 | learning rate: 7.997E-04 | global batch size: 512 | lm loss: 6.348343E+00 | loss scale: 4096.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.925 | TFLOPs: 72.45 | +[default7]: iteration 287/ 3814 | consumed samples: 146944 | consumed tokens: 300941312 | elapsed time per iteration (s): 5.35 | learning rate: 8.025E-04 | global batch size: 512 | lm loss: 6.353073E+00 | loss scale: 4096.0 | grad norm: 0.205 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.761 | TFLOPs: 72.33 | +[default7]: iteration 288/ 3814 | consumed samples: 147456 | consumed tokens: 301989888 | elapsed time per iteration (s): 5.33 | learning rate: 8.053E-04 | global batch size: 512 | lm loss: 6.354375E+00 | loss scale: 4096.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.044 | TFLOPs: 72.54 | +[default7]: iteration 289/ 3814 | consumed samples: 147968 | consumed tokens: 303038464 | elapsed time per iteration (s): 5.34 | learning rate: 8.081E-04 | global batch size: 512 | lm loss: 6.342794E+00 | loss scale: 4096.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.929 | TFLOPs: 72.46 | +[default7]: iteration 290/ 3814 | consumed samples: 148480 | consumed tokens: 304087040 | elapsed time per iteration (s): 5.34 | learning rate: 8.109E-04 | global batch size: 512 | lm loss: 6.329841E+00 | loss scale: 4096.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.808 | TFLOPs: 72.36 | +[default7]: iteration 291/ 3814 | consumed samples: 148992 | consumed tokens: 305135616 | elapsed time per iteration (s): 5.33 | learning rate: 8.137E-04 | global batch size: 512 | lm loss: 6.313782E+00 | loss scale: 4096.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.084 | TFLOPs: 72.57 | +[default7]: iteration 292/ 3814 | consumed samples: 149504 | consumed tokens: 306184192 | elapsed time per iteration (s): 5.34 | learning rate: 8.165E-04 | global batch size: 512 | lm loss: 6.322865E+00 | loss scale: 4096.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.967 | TFLOPs: 72.49 | +[default7]: iteration 293/ 3814 | consumed samples: 150016 | consumed tokens: 307232768 | elapsed time per iteration (s): 5.32 | learning rate: 8.193E-04 | global batch size: 512 | lm loss: 6.292122E+00 | loss scale: 4096.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.239 | TFLOPs: 72.69 | +[default7]: iteration 294/ 3814 | consumed samples: 150528 | consumed tokens: 308281344 | elapsed time per iteration (s): 5.32 | learning rate: 8.221E-04 | global batch size: 512 | lm loss: 6.264013E+00 | loss scale: 4096.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.170 | TFLOPs: 72.64 | +[default7]: iteration 295/ 3814 | consumed samples: 151040 | consumed tokens: 309329920 | elapsed time per iteration (s): 5.34 | learning rate: 8.249E-04 | global batch size: 512 | lm loss: 6.286059E+00 | loss scale: 4096.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.952 | TFLOPs: 72.47 | +[default7]: iteration 296/ 3814 | consumed samples: 151552 | consumed tokens: 310378496 | elapsed time per iteration (s): 5.33 | learning rate: 8.277E-04 | global batch size: 512 | lm loss: 6.264425E+00 | loss scale: 4096.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.061 | TFLOPs: 72.56 | +[default7]: iteration 297/ 3814 | consumed samples: 152064 | consumed tokens: 311427072 | elapsed time per iteration (s): 5.34 | learning rate: 8.305E-04 | global batch size: 512 | lm loss: 6.303043E+00 | loss scale: 4096.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.801 | TFLOPs: 72.36 | +[default7]: iteration 298/ 3814 | consumed samples: 152576 | consumed tokens: 312475648 | elapsed time per iteration (s): 5.35 | learning rate: 8.333E-04 | global batch size: 512 | lm loss: 6.326302E+00 | loss scale: 4096.0 | grad norm: 0.151 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.790 | TFLOPs: 72.35 | +[default7]: iteration 299/ 3814 | consumed samples: 153088 | consumed tokens: 313524224 | elapsed time per iteration (s): 5.34 | learning rate: 8.361E-04 | global batch size: 512 | lm loss: 6.289969E+00 | loss scale: 4096.0 | grad norm: 0.203 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.806 | TFLOPs: 72.36 | +[default7]: iteration 300/ 3814 | consumed samples: 153600 | consumed tokens: 314572800 | elapsed time per iteration (s): 5.35 | learning rate: 8.389E-04 | global batch size: 512 | lm loss: 6.297258E+00 | loss scale: 4096.0 | grad norm: 0.239 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.790 | TFLOPs: 72.35 | +[default7]: iteration 301/ 3814 | consumed samples: 154112 | consumed tokens: 315621376 | elapsed time per iteration (s): 5.34 | learning rate: 8.417E-04 | global batch size: 512 | lm loss: 6.320114E+00 | loss scale: 4096.0 | grad norm: 0.304 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.912 | TFLOPs: 72.44 | +[default7]: iteration 302/ 3814 | consumed samples: 154624 | consumed tokens: 316669952 | elapsed time per iteration (s): 5.33 | learning rate: 8.445E-04 | global batch size: 512 | lm loss: 6.295877E+00 | loss scale: 4096.0 | grad norm: 0.275 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.074 | TFLOPs: 72.57 | +[default7]: iteration 303/ 3814 | consumed samples: 155136 | consumed tokens: 317718528 | elapsed time per iteration (s): 5.34 | learning rate: 8.473E-04 | global batch size: 512 | lm loss: 6.273364E+00 | loss scale: 4096.0 | grad norm: 0.243 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.928 | TFLOPs: 72.46 | +[default7]: iteration 304/ 3814 | consumed samples: 155648 | consumed tokens: 318767104 | elapsed time per iteration (s): 5.34 | learning rate: 8.500E-04 | global batch size: 512 | lm loss: 6.306490E+00 | loss scale: 4096.0 | grad norm: 0.266 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.858 | TFLOPs: 72.40 | +[default7]: iteration 305/ 3814 | consumed samples: 156160 | consumed tokens: 319815680 | elapsed time per iteration (s): 5.34 | learning rate: 8.528E-04 | global batch size: 512 | lm loss: 6.272197E+00 | loss scale: 4096.0 | grad norm: 0.199 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.942 | TFLOPs: 72.47 | +[default7]: iteration 306/ 3814 | consumed samples: 156672 | consumed tokens: 320864256 | elapsed time per iteration (s): 5.34 | learning rate: 8.556E-04 | global batch size: 512 | lm loss: 6.234326E+00 | loss scale: 4096.0 | grad norm: 0.170 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.965 | TFLOPs: 72.48 | +[default7]: iteration 307/ 3814 | consumed samples: 157184 | consumed tokens: 321912832 | elapsed time per iteration (s): 5.38 | learning rate: 8.584E-04 | global batch size: 512 | lm loss: 6.240245E+00 | loss scale: 4096.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.177 | TFLOPs: 71.89 | +[default7]: iteration 308/ 3814 | consumed samples: 157696 | consumed tokens: 322961408 | elapsed time per iteration (s): 5.36 | learning rate: 8.612E-04 | global batch size: 512 | lm loss: 6.241629E+00 | loss scale: 4096.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.570 | TFLOPs: 72.19 | +[default7]: iteration 309/ 3814 | consumed samples: 158208 | consumed tokens: 324009984 | elapsed time per iteration (s): 5.36 | learning rate: 8.640E-04 | global batch size: 512 | lm loss: 6.232613E+00 | loss scale: 4096.0 | grad norm: 0.165 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.532 | TFLOPs: 72.16 | +[default7]: iteration 310/ 3814 | consumed samples: 158720 | consumed tokens: 325058560 | elapsed time per iteration (s): 5.35 | learning rate: 8.668E-04 | global batch size: 512 | lm loss: 6.215018E+00 | loss scale: 4096.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.679 | TFLOPs: 72.27 | +[default7]: iteration 311/ 3814 | consumed samples: 159232 | consumed tokens: 326107136 | elapsed time per iteration (s): 5.34 | learning rate: 8.696E-04 | global batch size: 512 | lm loss: 6.220246E+00 | loss scale: 4096.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.962 | TFLOPs: 72.48 | +[default7]: iteration 312/ 3814 | consumed samples: 159744 | consumed tokens: 327155712 | elapsed time per iteration (s): 5.33 | learning rate: 8.724E-04 | global batch size: 512 | lm loss: 6.219932E+00 | loss scale: 4096.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.044 | TFLOPs: 72.54 | +[default7]: iteration 313/ 3814 | consumed samples: 160256 | consumed tokens: 328204288 | elapsed time per iteration (s): 5.33 | learning rate: 8.752E-04 | global batch size: 512 | lm loss: 6.195639E+00 | loss scale: 4096.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.097 | TFLOPs: 72.58 | +[default7]: iteration 314/ 3814 | consumed samples: 160768 | consumed tokens: 329252864 | elapsed time per iteration (s): 5.34 | learning rate: 8.780E-04 | global batch size: 512 | lm loss: 6.188982E+00 | loss scale: 4096.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.803 | TFLOPs: 72.36 | +[default7]: iteration 315/ 3814 | consumed samples: 161280 | consumed tokens: 330301440 | elapsed time per iteration (s): 5.36 | learning rate: 8.808E-04 | global batch size: 512 | lm loss: 6.165828E+00 | loss scale: 4096.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.558 | TFLOPs: 72.18 | +[default7]: iteration 316/ 3814 | consumed samples: 161792 | consumed tokens: 331350016 | elapsed time per iteration (s): 5.34 | learning rate: 8.836E-04 | global batch size: 512 | lm loss: 6.160304E+00 | loss scale: 4096.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.911 | TFLOPs: 72.44 | +[default7]: iteration 317/ 3814 | consumed samples: 162304 | consumed tokens: 332398592 | elapsed time per iteration (s): 5.34 | learning rate: 8.864E-04 | global batch size: 512 | lm loss: 6.173937E+00 | loss scale: 4096.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.848 | TFLOPs: 72.39 | +[default7]: iteration 318/ 3814 | consumed samples: 162816 | consumed tokens: 333447168 | elapsed time per iteration (s): 5.37 | learning rate: 8.892E-04 | global batch size: 512 | lm loss: 6.185702E+00 | loss scale: 4096.0 | grad norm: 0.202 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.339 | TFLOPs: 72.01 | +[default7]: iteration 319/ 3814 | consumed samples: 163328 | consumed tokens: 334495744 | elapsed time per iteration (s): 5.34 | learning rate: 8.920E-04 | global batch size: 512 | lm loss: 6.194145E+00 | loss scale: 4096.0 | grad norm: 0.333 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.877 | TFLOPs: 72.42 | +[default7]: iteration 320/ 3814 | consumed samples: 163840 | consumed tokens: 335544320 | elapsed time per iteration (s): 5.34 | learning rate: 8.948E-04 | global batch size: 512 | lm loss: 6.256456E+00 | loss scale: 4096.0 | grad norm: 0.446 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.818 | TFLOPs: 72.37 | +[default7]: iteration 321/ 3814 | consumed samples: 164352 | consumed tokens: 336592896 | elapsed time per iteration (s): 5.33 | learning rate: 8.976E-04 | global batch size: 512 | lm loss: 6.255229E+00 | loss scale: 4096.0 | grad norm: 0.357 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.984 | TFLOPs: 72.50 | +[default7]: iteration 322/ 3814 | consumed samples: 164864 | consumed tokens: 337641472 | elapsed time per iteration (s): 5.33 | learning rate: 9.004E-04 | global batch size: 512 | lm loss: 6.215326E+00 | loss scale: 4096.0 | grad norm: 0.282 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.985 | TFLOPs: 72.50 | +[default7]: iteration 323/ 3814 | consumed samples: 165376 | consumed tokens: 338690048 | elapsed time per iteration (s): 5.35 | learning rate: 9.032E-04 | global batch size: 512 | lm loss: 6.201697E+00 | loss scale: 4096.0 | grad norm: 0.205 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.778 | TFLOPs: 72.34 | +[default7]: iteration 324/ 3814 | consumed samples: 165888 | consumed tokens: 339738624 | elapsed time per iteration (s): 5.35 | learning rate: 9.060E-04 | global batch size: 512 | lm loss: 6.165731E+00 | loss scale: 4096.0 | grad norm: 0.233 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.745 | TFLOPs: 72.32 | +[default7]: iteration 325/ 3814 | consumed samples: 166400 | consumed tokens: 340787200 | elapsed time per iteration (s): 5.35 | learning rate: 9.088E-04 | global batch size: 512 | lm loss: 6.177421E+00 | loss scale: 4096.0 | grad norm: 0.168 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.726 | TFLOPs: 72.30 | +[default7]: iteration 326/ 3814 | consumed samples: 166912 | consumed tokens: 341835776 | elapsed time per iteration (s): 5.33 | learning rate: 9.116E-04 | global batch size: 512 | lm loss: 6.173531E+00 | loss scale: 4096.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.015 | TFLOPs: 72.52 | +[default7]: iteration 327/ 3814 | consumed samples: 167424 | consumed tokens: 342884352 | elapsed time per iteration (s): 5.33 | learning rate: 9.144E-04 | global batch size: 512 | lm loss: 6.187306E+00 | loss scale: 4096.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.093 | TFLOPs: 72.58 | +[default7]: iteration 328/ 3814 | consumed samples: 167936 | consumed tokens: 343932928 | elapsed time per iteration (s): 5.35 | learning rate: 9.172E-04 | global batch size: 512 | lm loss: 6.178374E+00 | loss scale: 4096.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.786 | TFLOPs: 72.35 | +[default7]: iteration 329/ 3814 | consumed samples: 168448 | consumed tokens: 344981504 | elapsed time per iteration (s): 5.34 | learning rate: 9.200E-04 | global batch size: 512 | lm loss: 6.132145E+00 | loss scale: 4096.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.855 | TFLOPs: 72.40 | +[default7]: iteration 330/ 3814 | consumed samples: 168960 | consumed tokens: 346030080 | elapsed time per iteration (s): 5.35 | learning rate: 9.227E-04 | global batch size: 512 | lm loss: 6.142992E+00 | loss scale: 4096.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.663 | TFLOPs: 72.26 | +[default7]: iteration 331/ 3814 | consumed samples: 169472 | consumed tokens: 347078656 | elapsed time per iteration (s): 5.34 | learning rate: 9.255E-04 | global batch size: 512 | lm loss: 6.131600E+00 | loss scale: 4096.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.825 | TFLOPs: 72.38 | +[default7]: iteration 332/ 3814 | consumed samples: 169984 | consumed tokens: 348127232 | elapsed time per iteration (s): 5.35 | learning rate: 9.283E-04 | global batch size: 512 | lm loss: 6.103051E+00 | loss scale: 4096.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.757 | TFLOPs: 72.33 | +[default7]: iteration 333/ 3814 | consumed samples: 170496 | consumed tokens: 349175808 | elapsed time per iteration (s): 5.35 | learning rate: 9.311E-04 | global batch size: 512 | lm loss: 6.150748E+00 | loss scale: 4096.0 | grad norm: 0.239 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.678 | TFLOPs: 72.27 | +[default7]: iteration 334/ 3814 | consumed samples: 171008 | consumed tokens: 350224384 | elapsed time per iteration (s): 5.35 | learning rate: 9.339E-04 | global batch size: 512 | lm loss: 6.166262E+00 | loss scale: 4096.0 | grad norm: 0.303 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.696 | TFLOPs: 72.28 | +[default7]: iteration 335/ 3814 | consumed samples: 171520 | consumed tokens: 351272960 | elapsed time per iteration (s): 5.35 | learning rate: 9.367E-04 | global batch size: 512 | lm loss: 6.095911E+00 | loss scale: 4096.0 | grad norm: 0.162 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.634 | TFLOPs: 72.23 | +[default7]: iteration 336/ 3814 | consumed samples: 172032 | consumed tokens: 352321536 | elapsed time per iteration (s): 5.34 | learning rate: 9.395E-04 | global batch size: 512 | lm loss: 6.110657E+00 | loss scale: 4096.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.948 | TFLOPs: 72.47 | +[default7]: iteration 337/ 3814 | consumed samples: 172544 | consumed tokens: 353370112 | elapsed time per iteration (s): 5.33 | learning rate: 9.423E-04 | global batch size: 512 | lm loss: 6.086544E+00 | loss scale: 4096.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.067 | TFLOPs: 72.56 | +[default7]: iteration 338/ 3814 | consumed samples: 173056 | consumed tokens: 354418688 | elapsed time per iteration (s): 5.32 | learning rate: 9.451E-04 | global batch size: 512 | lm loss: 6.077362E+00 | loss scale: 4096.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.234 | TFLOPs: 72.69 | +[default7]: iteration 339/ 3814 | consumed samples: 173568 | consumed tokens: 355467264 | elapsed time per iteration (s): 5.31 | learning rate: 9.479E-04 | global batch size: 512 | lm loss: 6.092340E+00 | loss scale: 4096.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.452 | TFLOPs: 72.85 | +[default7]: iteration 340/ 3814 | consumed samples: 174080 | consumed tokens: 356515840 | elapsed time per iteration (s): 5.32 | learning rate: 9.507E-04 | global batch size: 512 | lm loss: 6.061146E+00 | loss scale: 4096.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.174 | TFLOPs: 72.64 | +[default7]: iteration 341/ 3814 | consumed samples: 174592 | consumed tokens: 357564416 | elapsed time per iteration (s): 5.35 | learning rate: 9.535E-04 | global batch size: 512 | lm loss: 6.093139E+00 | loss scale: 4096.0 | grad norm: 0.186 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.783 | TFLOPs: 72.35 | +[default7]: iteration 342/ 3814 | consumed samples: 175104 | consumed tokens: 358612992 | elapsed time per iteration (s): 5.32 | learning rate: 9.563E-04 | global batch size: 512 | lm loss: 6.077615E+00 | loss scale: 4096.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.290 | TFLOPs: 72.73 | +[default7]: iteration 343/ 3814 | consumed samples: 175616 | consumed tokens: 359661568 | elapsed time per iteration (s): 5.38 | learning rate: 9.591E-04 | global batch size: 512 | lm loss: 6.051147E+00 | loss scale: 4096.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.130 | TFLOPs: 71.85 | +[default7]: iteration 344/ 3814 | consumed samples: 176128 | consumed tokens: 360710144 | elapsed time per iteration (s): 5.35 | learning rate: 9.619E-04 | global batch size: 512 | lm loss: 6.065012E+00 | loss scale: 4096.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.784 | TFLOPs: 72.35 | +[default7]: iteration 345/ 3814 | consumed samples: 176640 | consumed tokens: 361758720 | elapsed time per iteration (s): 5.34 | learning rate: 9.647E-04 | global batch size: 512 | lm loss: 6.063989E+00 | loss scale: 4096.0 | grad norm: 0.212 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.896 | TFLOPs: 72.43 | +[default7]: iteration 346/ 3814 | consumed samples: 177152 | consumed tokens: 362807296 | elapsed time per iteration (s): 5.35 | learning rate: 9.675E-04 | global batch size: 512 | lm loss: 6.076988E+00 | loss scale: 4096.0 | grad norm: 0.324 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.771 | TFLOPs: 72.34 | +[default7]: iteration 347/ 3814 | consumed samples: 177664 | consumed tokens: 363855872 | elapsed time per iteration (s): 5.34 | learning rate: 9.703E-04 | global batch size: 512 | lm loss: 6.072388E+00 | loss scale: 4096.0 | grad norm: 0.344 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.846 | TFLOPs: 72.39 | +[default7]: iteration 348/ 3814 | consumed samples: 178176 | consumed tokens: 364904448 | elapsed time per iteration (s): 5.34 | learning rate: 9.731E-04 | global batch size: 512 | lm loss: 6.122403E+00 | loss scale: 4096.0 | grad norm: 0.355 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.896 | TFLOPs: 72.43 | +[default7]: iteration 349/ 3814 | consumed samples: 178688 | consumed tokens: 365953024 | elapsed time per iteration (s): 5.36 | learning rate: 9.759E-04 | global batch size: 512 | lm loss: 6.117064E+00 | loss scale: 4096.0 | grad norm: 0.372 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.453 | TFLOPs: 72.10 | +[default7]: iteration 350/ 3814 | consumed samples: 179200 | consumed tokens: 367001600 | elapsed time per iteration (s): 5.36 | learning rate: 9.787E-04 | global batch size: 512 | lm loss: 6.079517E+00 | loss scale: 4096.0 | grad norm: 0.312 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.486 | TFLOPs: 72.12 | +[default7]: iteration 351/ 3814 | consumed samples: 179712 | consumed tokens: 368050176 | elapsed time per iteration (s): 5.37 | learning rate: 9.815E-04 | global batch size: 512 | lm loss: 6.089908E+00 | loss scale: 4096.0 | grad norm: 0.181 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.407 | TFLOPs: 72.06 | +[default7]: iteration 352/ 3814 | consumed samples: 180224 | consumed tokens: 369098752 | elapsed time per iteration (s): 5.37 | learning rate: 9.843E-04 | global batch size: 512 | lm loss: 6.069890E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.420 | TFLOPs: 72.07 | +[default7]: iteration 353/ 3814 | consumed samples: 180736 | consumed tokens: 370147328 | elapsed time per iteration (s): 5.36 | learning rate: 9.871E-04 | global batch size: 512 | lm loss: 6.025175E+00 | loss scale: 4096.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.595 | TFLOPs: 72.20 | +[default7]: iteration 354/ 3814 | consumed samples: 181248 | consumed tokens: 371195904 | elapsed time per iteration (s): 5.35 | learning rate: 9.899E-04 | global batch size: 512 | lm loss: 6.037868E+00 | loss scale: 4096.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.674 | TFLOPs: 72.26 | +[default7]: iteration 355/ 3814 | consumed samples: 181760 | consumed tokens: 372244480 | elapsed time per iteration (s): 5.36 | learning rate: 9.927E-04 | global batch size: 512 | lm loss: 6.026511E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.545 | TFLOPs: 72.17 | +[default7]: iteration 356/ 3814 | consumed samples: 182272 | consumed tokens: 373293056 | elapsed time per iteration (s): 5.38 | learning rate: 9.955E-04 | global batch size: 512 | lm loss: 5.992759E+00 | loss scale: 4096.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.214 | TFLOPs: 71.92 | +[default7]: iteration 357/ 3814 | consumed samples: 182784 | consumed tokens: 374341632 | elapsed time per iteration (s): 5.36 | learning rate: 9.982E-04 | global batch size: 512 | lm loss: 6.013062E+00 | loss scale: 4096.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.547 | TFLOPs: 72.17 | +[default7]: iteration 358/ 3814 | consumed samples: 183296 | consumed tokens: 375390208 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.971052E+00 | loss scale: 4096.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.799 | TFLOPs: 72.36 | +[default7]: iteration 359/ 3814 | consumed samples: 183808 | consumed tokens: 376438784 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.958958E+00 | loss scale: 4096.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.907 | TFLOPs: 72.44 | +[default7]: iteration 360/ 3814 | consumed samples: 184320 | consumed tokens: 377487360 | elapsed time per iteration (s): 5.35 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.959209E+00 | loss scale: 4096.0 | grad norm: 0.185 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.751 | TFLOPs: 72.32 | +[default7]: iteration 361/ 3814 | consumed samples: 184832 | consumed tokens: 378535936 | elapsed time per iteration (s): 5.35 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.983677E+00 | loss scale: 4096.0 | grad norm: 0.284 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.699 | TFLOPs: 72.28 | +[default7]: iteration 362/ 3814 | consumed samples: 185344 | consumed tokens: 379584512 | elapsed time per iteration (s): 5.36 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.978217E+00 | loss scale: 4096.0 | grad norm: 0.212 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.530 | TFLOPs: 72.16 | +[default7]: iteration 363/ 3814 | consumed samples: 185856 | consumed tokens: 380633088 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.987855E+00 | loss scale: 4096.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.911 | TFLOPs: 72.44 | +[default7]: iteration 364/ 3814 | consumed samples: 186368 | consumed tokens: 381681664 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.966809E+00 | loss scale: 4096.0 | grad norm: 0.179 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.957 | TFLOPs: 72.48 | +[default7]: iteration 365/ 3814 | consumed samples: 186880 | consumed tokens: 382730240 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.967506E+00 | loss scale: 4096.0 | grad norm: 0.221 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.901 | TFLOPs: 72.44 | +[default7]: iteration 366/ 3814 | consumed samples: 187392 | consumed tokens: 383778816 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.969231E+00 | loss scale: 4096.0 | grad norm: 0.220 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.857 | TFLOPs: 72.40 | +[default7]: iteration 367/ 3814 | consumed samples: 187904 | consumed tokens: 384827392 | elapsed time per iteration (s): 5.35 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.959604E+00 | loss scale: 4096.0 | grad norm: 0.177 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.700 | TFLOPs: 72.28 | +[default7]: iteration 368/ 3814 | consumed samples: 188416 | consumed tokens: 385875968 | elapsed time per iteration (s): 5.33 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.922968E+00 | loss scale: 4096.0 | grad norm: 0.162 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.009 | TFLOPs: 72.52 | +[default7]: iteration 369/ 3814 | consumed samples: 188928 | consumed tokens: 386924544 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.931462E+00 | loss scale: 4096.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.935 | TFLOPs: 72.46 | +[default7]: iteration 370/ 3814 | consumed samples: 189440 | consumed tokens: 387973120 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.904366E+00 | loss scale: 4096.0 | grad norm: 0.148 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.869 | TFLOPs: 72.41 | +[default7]: iteration 371/ 3814 | consumed samples: 189952 | consumed tokens: 389021696 | elapsed time per iteration (s): 5.33 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.912099E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.025 | TFLOPs: 72.53 | +[default7]: iteration 372/ 3814 | consumed samples: 190464 | consumed tokens: 390070272 | elapsed time per iteration (s): 5.36 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.919347E+00 | loss scale: 4096.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.584 | TFLOPs: 72.20 | +[default7]: iteration 373/ 3814 | consumed samples: 190976 | consumed tokens: 391118848 | elapsed time per iteration (s): 5.35 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.923248E+00 | loss scale: 4096.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.635 | TFLOPs: 72.23 | +[default7]: iteration 374/ 3814 | consumed samples: 191488 | consumed tokens: 392167424 | elapsed time per iteration (s): 5.34 | learning rate: 1.000E-03 | global batch size: 512 | lm loss: 5.911174E+00 | loss scale: 4096.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.844 | TFLOPs: 72.39 | +[default7]: iteration 375/ 3814 | consumed samples: 192000 | consumed tokens: 393216000 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.904548E+00 | loss scale: 4096.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.907 | TFLOPs: 72.44 | +[default7]: iteration 376/ 3814 | consumed samples: 192512 | consumed tokens: 394264576 | elapsed time per iteration (s): 5.35 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.873360E+00 | loss scale: 4096.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.728 | TFLOPs: 72.30 | +[default7]: iteration 377/ 3814 | consumed samples: 193024 | consumed tokens: 395313152 | elapsed time per iteration (s): 5.35 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.859234E+00 | loss scale: 4096.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.725 | TFLOPs: 72.30 | +[default7]: iteration 378/ 3814 | consumed samples: 193536 | consumed tokens: 396361728 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.844400E+00 | loss scale: 4096.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.870 | TFLOPs: 72.41 | +[default7]: iteration 379/ 3814 | consumed samples: 194048 | consumed tokens: 397410304 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.899143E+00 | loss scale: 4096.0 | grad norm: 0.238 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.925 | TFLOPs: 72.45 | +[default7]: iteration 380/ 3814 | consumed samples: 194560 | consumed tokens: 398458880 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.907044E+00 | loss scale: 4096.0 | grad norm: 0.224 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.961 | TFLOPs: 72.48 | +[default7]: iteration 381/ 3814 | consumed samples: 195072 | consumed tokens: 399507456 | elapsed time per iteration (s): 5.35 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.887918E+00 | loss scale: 4096.0 | grad norm: 0.170 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.639 | TFLOPs: 72.24 | +[default7]: iteration 382/ 3814 | consumed samples: 195584 | consumed tokens: 400556032 | elapsed time per iteration (s): 5.33 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.867260E+00 | loss scale: 4096.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.051 | TFLOPs: 72.55 | +[default7]: iteration 383/ 3814 | consumed samples: 196096 | consumed tokens: 401604608 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.873287E+00 | loss scale: 4096.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.915 | TFLOPs: 72.45 | +[default7]: iteration 384/ 3814 | consumed samples: 196608 | consumed tokens: 402653184 | elapsed time per iteration (s): 5.34 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.879637E+00 | loss scale: 4096.0 | grad norm: 0.258 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.868 | TFLOPs: 72.41 | +[default7]: iteration 385/ 3814 | consumed samples: 197120 | consumed tokens: 403701760 | elapsed time per iteration (s): 5.38 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.883770E+00 | loss scale: 4096.0 | grad norm: 0.306 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.219 | TFLOPs: 71.92 | +[default7]: iteration 386/ 3814 | consumed samples: 197632 | consumed tokens: 404750336 | elapsed time per iteration (s): 5.35 | learning rate: 9.999E-04 | global batch size: 512 | lm loss: 5.898153E+00 | loss scale: 4096.0 | grad norm: 0.285 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.626 | TFLOPs: 72.23 | +[default7]: iteration 387/ 3814 | consumed samples: 198144 | consumed tokens: 405798912 | elapsed time per iteration (s): 5.35 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.868663E+00 | loss scale: 4096.0 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.727 | TFLOPs: 72.30 | +[default7]: iteration 388/ 3814 | consumed samples: 198656 | consumed tokens: 406847488 | elapsed time per iteration (s): 5.34 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.866872E+00 | loss scale: 4096.0 | grad norm: 0.192 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.810 | TFLOPs: 72.37 | +[default7]: iteration 389/ 3814 | consumed samples: 199168 | consumed tokens: 407896064 | elapsed time per iteration (s): 5.33 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.833775E+00 | loss scale: 4096.0 | grad norm: 0.199 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.078 | TFLOPs: 72.57 | +[default7]: iteration 390/ 3814 | consumed samples: 199680 | consumed tokens: 408944640 | elapsed time per iteration (s): 5.33 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.857618E+00 | loss scale: 4096.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.978 | TFLOPs: 72.49 | +[default7]: iteration 391/ 3814 | consumed samples: 200192 | consumed tokens: 409993216 | elapsed time per iteration (s): 5.35 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.857711E+00 | loss scale: 4096.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.701 | TFLOPs: 72.28 | +[default7]: iteration 392/ 3814 | consumed samples: 200704 | consumed tokens: 411041792 | elapsed time per iteration (s): 5.34 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.791764E+00 | loss scale: 4096.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.854 | TFLOPs: 72.40 | +[default7]: iteration 393/ 3814 | consumed samples: 201216 | consumed tokens: 412090368 | elapsed time per iteration (s): 5.33 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.819961E+00 | loss scale: 4096.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.114 | TFLOPs: 72.60 | +[default7]: iteration 394/ 3814 | consumed samples: 201728 | consumed tokens: 413138944 | elapsed time per iteration (s): 5.33 | learning rate: 9.998E-04 | global batch size: 512 | lm loss: 5.819483E+00 | loss scale: 4096.0 | grad norm: 0.151 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.055 | TFLOPs: 72.55 | +[default7]: iteration 395/ 3814 | consumed samples: 202240 | consumed tokens: 414187520 | elapsed time per iteration (s): 5.34 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.811018E+00 | loss scale: 4096.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.964 | TFLOPs: 72.48 | +[default7]: iteration 396/ 3814 | consumed samples: 202752 | consumed tokens: 415236096 | elapsed time per iteration (s): 5.32 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.800707E+00 | loss scale: 4096.0 | grad norm: 0.192 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.291 | TFLOPs: 72.73 | +[default7]: iteration 397/ 3814 | consumed samples: 203264 | consumed tokens: 416284672 | elapsed time per iteration (s): 5.31 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.823111E+00 | loss scale: 4096.0 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.342 | TFLOPs: 72.77 | +[default7]: iteration 398/ 3814 | consumed samples: 203776 | consumed tokens: 417333248 | elapsed time per iteration (s): 5.35 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.811250E+00 | loss scale: 4096.0 | grad norm: 0.227 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.639 | TFLOPs: 72.24 | +[default7]: iteration 399/ 3814 | consumed samples: 204288 | consumed tokens: 418381824 | elapsed time per iteration (s): 5.34 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.789074E+00 | loss scale: 4096.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.797 | TFLOPs: 72.36 | +[default7]: iteration 400/ 3814 | consumed samples: 204800 | consumed tokens: 419430400 | elapsed time per iteration (s): 5.35 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.780504E+00 | loss scale: 4096.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.770 | TFLOPs: 72.34 | +[default7]: iteration 401/ 3814 | consumed samples: 205312 | consumed tokens: 420478976 | elapsed time per iteration (s): 5.35 | learning rate: 9.997E-04 | global batch size: 512 | lm loss: 5.773182E+00 | loss scale: 4096.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.742 | TFLOPs: 72.31 | +[default7]: iteration 402/ 3814 | consumed samples: 205824 | consumed tokens: 421527552 | elapsed time per iteration (s): 5.35 | learning rate: 9.996E-04 | global batch size: 512 | lm loss: 5.771012E+00 | loss scale: 4096.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.788 | TFLOPs: 72.35 | +[default7]: iteration 403/ 3814 | consumed samples: 206336 | consumed tokens: 422576128 | elapsed time per iteration (s): 5.35 | learning rate: 9.996E-04 | global batch size: 512 | lm loss: 5.763574E+00 | loss scale: 4096.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.754 | TFLOPs: 72.32 | +[default7]: iteration 404/ 3814 | consumed samples: 206848 | consumed tokens: 423624704 | elapsed time per iteration (s): 5.36 | learning rate: 9.996E-04 | global batch size: 512 | lm loss: 5.735404E+00 | loss scale: 4096.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.610 | TFLOPs: 72.22 | +[default7]: iteration 405/ 3814 | consumed samples: 207360 | consumed tokens: 424673280 | elapsed time per iteration (s): 5.35 | learning rate: 9.996E-04 | global batch size: 512 | lm loss: 5.744381E+00 | loss scale: 4096.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.615 | TFLOPs: 72.22 | +[default7]: iteration 406/ 3814 | consumed samples: 207872 | consumed tokens: 425721856 | elapsed time per iteration (s): 5.33 | learning rate: 9.996E-04 | global batch size: 512 | lm loss: 5.740142E+00 | loss scale: 4096.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.027 | TFLOPs: 72.53 | +[default7]: iteration 407/ 3814 | consumed samples: 208384 | consumed tokens: 426770432 | elapsed time per iteration (s): 5.35 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.751395E+00 | loss scale: 4096.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.688 | TFLOPs: 72.27 | +[default7]: iteration 408/ 3814 | consumed samples: 208896 | consumed tokens: 427819008 | elapsed time per iteration (s): 5.35 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.725862E+00 | loss scale: 4096.0 | grad norm: 0.187 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.734 | TFLOPs: 72.31 | +[default7]: iteration 409/ 3814 | consumed samples: 209408 | consumed tokens: 428867584 | elapsed time per iteration (s): 5.35 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.737408E+00 | loss scale: 4096.0 | grad norm: 0.223 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.685 | TFLOPs: 72.27 | +[default7]: iteration 410/ 3814 | consumed samples: 209920 | consumed tokens: 429916160 | elapsed time per iteration (s): 5.34 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.716256E+00 | loss scale: 4096.0 | grad norm: 0.190 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.871 | TFLOPs: 72.41 | +[default7]: iteration 411/ 3814 | consumed samples: 210432 | consumed tokens: 430964736 | elapsed time per iteration (s): 5.37 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.700470E+00 | loss scale: 4096.0 | grad norm: 0.197 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.393 | TFLOPs: 72.05 | +[default7]: iteration 412/ 3814 | consumed samples: 210944 | consumed tokens: 432013312 | elapsed time per iteration (s): 5.34 | learning rate: 9.995E-04 | global batch size: 512 | lm loss: 5.733970E+00 | loss scale: 4096.0 | grad norm: 0.295 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.820 | TFLOPs: 72.37 | +[default7]: iteration 413/ 3814 | consumed samples: 211456 | consumed tokens: 433061888 | elapsed time per iteration (s): 5.36 | learning rate: 9.994E-04 | global batch size: 512 | lm loss: 5.738263E+00 | loss scale: 4096.0 | grad norm: 0.223 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.462 | TFLOPs: 72.10 | +[default7]: iteration 414/ 3814 | consumed samples: 211968 | consumed tokens: 434110464 | elapsed time per iteration (s): 5.36 | learning rate: 9.994E-04 | global batch size: 512 | lm loss: 5.707120E+00 | loss scale: 4096.0 | grad norm: 0.219 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.549 | TFLOPs: 72.17 | +[default7]: iteration 415/ 3814 | consumed samples: 212480 | consumed tokens: 435159040 | elapsed time per iteration (s): 5.34 | learning rate: 9.994E-04 | global batch size: 512 | lm loss: 5.725977E+00 | loss scale: 4096.0 | grad norm: 0.300 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.896 | TFLOPs: 72.43 | +[default7]: iteration 416/ 3814 | consumed samples: 212992 | consumed tokens: 436207616 | elapsed time per iteration (s): 5.37 | learning rate: 9.994E-04 | global batch size: 512 | lm loss: 5.738180E+00 | loss scale: 4096.0 | grad norm: 0.272 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.381 | TFLOPs: 72.04 | +[default7]: iteration 417/ 3814 | consumed samples: 213504 | consumed tokens: 437256192 | elapsed time per iteration (s): 5.36 | learning rate: 9.993E-04 | global batch size: 512 | lm loss: 5.714120E+00 | loss scale: 4096.0 | grad norm: 0.230 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.458 | TFLOPs: 72.10 | +[default7]: iteration 418/ 3814 | consumed samples: 214016 | consumed tokens: 438304768 | elapsed time per iteration (s): 5.35 | learning rate: 9.993E-04 | global batch size: 512 | lm loss: 5.709915E+00 | loss scale: 4096.0 | grad norm: 0.235 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.651 | TFLOPs: 72.25 | +[default7]: iteration 419/ 3814 | consumed samples: 214528 | consumed tokens: 439353344 | elapsed time per iteration (s): 5.37 | learning rate: 9.993E-04 | global batch size: 512 | lm loss: 5.724755E+00 | loss scale: 4096.0 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.370 | TFLOPs: 72.03 | +[default7]: iteration 420/ 3814 | consumed samples: 215040 | consumed tokens: 440401920 | elapsed time per iteration (s): 5.36 | learning rate: 9.993E-04 | global batch size: 512 | lm loss: 5.726122E+00 | loss scale: 4096.0 | grad norm: 0.221 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.546 | TFLOPs: 72.17 | +[default7]: iteration 421/ 3814 | consumed samples: 215552 | consumed tokens: 441450496 | elapsed time per iteration (s): 5.37 | learning rate: 9.993E-04 | global batch size: 512 | lm loss: 5.700251E+00 | loss scale: 4096.0 | grad norm: 0.160 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.354 | TFLOPs: 72.02 | +[default7]: iteration 422/ 3814 | consumed samples: 216064 | consumed tokens: 442499072 | elapsed time per iteration (s): 5.38 | learning rate: 9.992E-04 | global batch size: 512 | lm loss: 5.692668E+00 | loss scale: 4096.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.150 | TFLOPs: 71.87 | +[default7]: iteration 423/ 3814 | consumed samples: 216576 | consumed tokens: 443547648 | elapsed time per iteration (s): 5.36 | learning rate: 9.992E-04 | global batch size: 512 | lm loss: 5.666317E+00 | loss scale: 4096.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.523 | TFLOPs: 72.15 | +[default7]: iteration 424/ 3814 | consumed samples: 217088 | consumed tokens: 444596224 | elapsed time per iteration (s): 5.37 | learning rate: 9.992E-04 | global batch size: 512 | lm loss: 5.697589E+00 | loss scale: 4096.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.382 | TFLOPs: 72.04 | +[default7]: iteration 425/ 3814 | consumed samples: 217600 | consumed tokens: 445644800 | elapsed time per iteration (s): 5.37 | learning rate: 9.992E-04 | global batch size: 512 | lm loss: 5.663877E+00 | loss scale: 4096.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.383 | TFLOPs: 72.04 | +[default7]: iteration 426/ 3814 | consumed samples: 218112 | consumed tokens: 446693376 | elapsed time per iteration (s): 5.36 | learning rate: 9.991E-04 | global batch size: 512 | lm loss: 5.652926E+00 | loss scale: 4096.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.556 | TFLOPs: 72.17 | +[default7]: iteration 427/ 3814 | consumed samples: 218624 | consumed tokens: 447741952 | elapsed time per iteration (s): 5.36 | learning rate: 9.991E-04 | global batch size: 512 | lm loss: 5.647188E+00 | loss scale: 4096.0 | grad norm: 0.156 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.444 | TFLOPs: 72.09 | +[default7]: iteration 428/ 3814 | consumed samples: 219136 | consumed tokens: 448790528 | elapsed time per iteration (s): 5.35 | learning rate: 9.991E-04 | global batch size: 512 | lm loss: 5.652099E+00 | loss scale: 4096.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.667 | TFLOPs: 72.26 | +[default7]: iteration 429/ 3814 | consumed samples: 219648 | consumed tokens: 449839104 | elapsed time per iteration (s): 5.37 | learning rate: 9.991E-04 | global batch size: 512 | lm loss: 5.637729E+00 | loss scale: 4096.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.259 | TFLOPs: 71.95 | +[default7]: iteration 430/ 3814 | consumed samples: 220160 | consumed tokens: 450887680 | elapsed time per iteration (s): 5.35 | learning rate: 9.990E-04 | global batch size: 512 | lm loss: 5.640182E+00 | loss scale: 4096.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.759 | TFLOPs: 72.33 | +[default7]: iteration 431/ 3814 | consumed samples: 220672 | consumed tokens: 451936256 | elapsed time per iteration (s): 5.35 | learning rate: 9.990E-04 | global batch size: 512 | lm loss: 5.583520E+00 | loss scale: 4096.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.627 | TFLOPs: 72.23 | +[default7]: iteration 432/ 3814 | consumed samples: 221184 | consumed tokens: 452984832 | elapsed time per iteration (s): 5.36 | learning rate: 9.990E-04 | global batch size: 512 | lm loss: 5.607174E+00 | loss scale: 4096.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.457 | TFLOPs: 72.10 | +[default7]: iteration 433/ 3814 | consumed samples: 221696 | consumed tokens: 454033408 | elapsed time per iteration (s): 5.37 | learning rate: 9.989E-04 | global batch size: 512 | lm loss: 5.599467E+00 | loss scale: 4096.0 | grad norm: 0.171 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.314 | TFLOPs: 71.99 | +[default7]: iteration 434/ 3814 | consumed samples: 222208 | consumed tokens: 455081984 | elapsed time per iteration (s): 5.37 | learning rate: 9.989E-04 | global batch size: 512 | lm loss: 5.613785E+00 | loss scale: 4096.0 | grad norm: 0.194 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.379 | TFLOPs: 72.04 | +[default7]: iteration 435/ 3814 | consumed samples: 222720 | consumed tokens: 456130560 | elapsed time per iteration (s): 5.35 | learning rate: 9.989E-04 | global batch size: 512 | lm loss: 5.616014E+00 | loss scale: 4096.0 | grad norm: 0.173 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.728 | TFLOPs: 72.30 | +[default7]: iteration 436/ 3814 | consumed samples: 223232 | consumed tokens: 457179136 | elapsed time per iteration (s): 5.37 | learning rate: 9.989E-04 | global batch size: 512 | lm loss: 5.627961E+00 | loss scale: 4096.0 | grad norm: 0.158 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.380 | TFLOPs: 72.04 | +[default7]: iteration 437/ 3814 | consumed samples: 223744 | consumed tokens: 458227712 | elapsed time per iteration (s): 5.37 | learning rate: 9.988E-04 | global batch size: 512 | lm loss: 5.587533E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.326 | TFLOPs: 72.00 | +[default7]: iteration 438/ 3814 | consumed samples: 224256 | consumed tokens: 459276288 | elapsed time per iteration (s): 5.37 | learning rate: 9.988E-04 | global batch size: 512 | lm loss: 5.606798E+00 | loss scale: 4096.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.364 | TFLOPs: 72.03 | +[default7]: iteration 439/ 3814 | consumed samples: 224768 | consumed tokens: 460324864 | elapsed time per iteration (s): 5.35 | learning rate: 9.988E-04 | global batch size: 512 | lm loss: 5.609087E+00 | loss scale: 4096.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.657 | TFLOPs: 72.25 | +[default7]: iteration 440/ 3814 | consumed samples: 225280 | consumed tokens: 461373440 | elapsed time per iteration (s): 5.35 | learning rate: 9.987E-04 | global batch size: 512 | lm loss: 5.576598E+00 | loss scale: 4096.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.711 | TFLOPs: 72.29 | +[default7]: iteration 441/ 3814 | consumed samples: 225792 | consumed tokens: 462422016 | elapsed time per iteration (s): 5.36 | learning rate: 9.987E-04 | global batch size: 512 | lm loss: 5.582748E+00 | loss scale: 4096.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.517 | TFLOPs: 72.15 | +[default7]: iteration 442/ 3814 | consumed samples: 226304 | consumed tokens: 463470592 | elapsed time per iteration (s): 5.36 | learning rate: 9.987E-04 | global batch size: 512 | lm loss: 5.597144E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.497 | TFLOPs: 72.13 | +[default7]: iteration 443/ 3814 | consumed samples: 226816 | consumed tokens: 464519168 | elapsed time per iteration (s): 5.36 | learning rate: 9.986E-04 | global batch size: 512 | lm loss: 5.561965E+00 | loss scale: 4096.0 | grad norm: 0.183 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.460 | TFLOPs: 72.10 | +[default7]: iteration 444/ 3814 | consumed samples: 227328 | consumed tokens: 465567744 | elapsed time per iteration (s): 5.36 | learning rate: 9.986E-04 | global batch size: 512 | lm loss: 5.583800E+00 | loss scale: 4096.0 | grad norm: 0.211 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.531 | TFLOPs: 72.16 | +[default7]: iteration 445/ 3814 | consumed samples: 227840 | consumed tokens: 466616320 | elapsed time per iteration (s): 5.35 | learning rate: 9.986E-04 | global batch size: 512 | lm loss: 5.586281E+00 | loss scale: 4096.0 | grad norm: 0.240 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.702 | TFLOPs: 72.28 | +[default7]: iteration 446/ 3814 | consumed samples: 228352 | consumed tokens: 467664896 | elapsed time per iteration (s): 5.36 | learning rate: 9.985E-04 | global batch size: 512 | lm loss: 5.616479E+00 | loss scale: 4096.0 | grad norm: 0.326 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.442 | TFLOPs: 72.09 | +[default7]: iteration 447/ 3814 | consumed samples: 228864 | consumed tokens: 468713472 | elapsed time per iteration (s): 5.38 | learning rate: 9.985E-04 | global batch size: 512 | lm loss: 5.598700E+00 | loss scale: 4096.0 | grad norm: 0.322 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.155 | TFLOPs: 71.87 | +[default7]: iteration 448/ 3814 | consumed samples: 229376 | consumed tokens: 469762048 | elapsed time per iteration (s): 5.36 | learning rate: 9.985E-04 | global batch size: 512 | lm loss: 5.632094E+00 | loss scale: 4096.0 | grad norm: 0.283 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.595 | TFLOPs: 72.20 | +[default7]: iteration 449/ 3814 | consumed samples: 229888 | consumed tokens: 470810624 | elapsed time per iteration (s): 5.37 | learning rate: 9.984E-04 | global batch size: 512 | lm loss: 5.551995E+00 | loss scale: 4096.0 | grad norm: 0.211 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.367 | TFLOPs: 72.03 | +[default7]: iteration 450/ 3814 | consumed samples: 230400 | consumed tokens: 471859200 | elapsed time per iteration (s): 5.39 | learning rate: 9.984E-04 | global batch size: 512 | lm loss: 5.581970E+00 | loss scale: 4096.0 | grad norm: 0.166 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.921 | TFLOPs: 71.70 | +[default7]: iteration 451/ 3814 | consumed samples: 230912 | consumed tokens: 472907776 | elapsed time per iteration (s): 5.37 | learning rate: 9.984E-04 | global batch size: 512 | lm loss: 5.568761E+00 | loss scale: 4096.0 | grad norm: 0.205 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.269 | TFLOPs: 71.96 | +[default7]: iteration 452/ 3814 | consumed samples: 231424 | consumed tokens: 473956352 | elapsed time per iteration (s): 5.37 | learning rate: 9.983E-04 | global batch size: 512 | lm loss: 5.571843E+00 | loss scale: 4096.0 | grad norm: 0.189 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.321 | TFLOPs: 72.00 | +[default7]: iteration 453/ 3814 | consumed samples: 231936 | consumed tokens: 475004928 | elapsed time per iteration (s): 5.35 | learning rate: 9.983E-04 | global batch size: 512 | lm loss: 5.555061E+00 | loss scale: 4096.0 | grad norm: 0.188 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.660 | TFLOPs: 72.25 | +[default7]: iteration 454/ 3814 | consumed samples: 232448 | consumed tokens: 476053504 | elapsed time per iteration (s): 5.37 | learning rate: 9.983E-04 | global batch size: 512 | lm loss: 5.536522E+00 | loss scale: 4096.0 | grad norm: 0.185 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.404 | TFLOPs: 72.06 | +[default7]: iteration 455/ 3814 | consumed samples: 232960 | consumed tokens: 477102080 | elapsed time per iteration (s): 5.38 | learning rate: 9.982E-04 | global batch size: 512 | lm loss: 5.537925E+00 | loss scale: 4096.0 | grad norm: 0.159 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.217 | TFLOPs: 71.92 | +[default7]: iteration 456/ 3814 | consumed samples: 233472 | consumed tokens: 478150656 | elapsed time per iteration (s): 5.37 | learning rate: 9.982E-04 | global batch size: 512 | lm loss: 5.549852E+00 | loss scale: 4096.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.290 | TFLOPs: 71.97 | +[default7]: iteration 457/ 3814 | consumed samples: 233984 | consumed tokens: 479199232 | elapsed time per iteration (s): 5.39 | learning rate: 9.982E-04 | global batch size: 512 | lm loss: 5.530914E+00 | loss scale: 4096.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.004 | TFLOPs: 71.76 | +[default7]: iteration 458/ 3814 | consumed samples: 234496 | consumed tokens: 480247808 | elapsed time per iteration (s): 5.33 | learning rate: 9.981E-04 | global batch size: 512 | lm loss: 5.540121E+00 | loss scale: 4096.0 | grad norm: 0.156 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.064 | TFLOPs: 72.56 | +[default7]: iteration 459/ 3814 | consumed samples: 235008 | consumed tokens: 481296384 | elapsed time per iteration (s): 5.36 | learning rate: 9.981E-04 | global batch size: 512 | lm loss: 5.478067E+00 | loss scale: 4096.0 | grad norm: 0.144 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.474 | TFLOPs: 72.11 | +[default7]: iteration 460/ 3814 | consumed samples: 235520 | consumed tokens: 482344960 | elapsed time per iteration (s): 5.36 | learning rate: 9.981E-04 | global batch size: 512 | lm loss: 5.517561E+00 | loss scale: 4096.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.456 | TFLOPs: 72.10 | +[default7]: iteration 461/ 3814 | consumed samples: 236032 | consumed tokens: 483393536 | elapsed time per iteration (s): 5.36 | learning rate: 9.980E-04 | global batch size: 512 | lm loss: 5.499029E+00 | loss scale: 4096.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.455 | TFLOPs: 72.10 | +[default7]: iteration 462/ 3814 | consumed samples: 236544 | consumed tokens: 484442112 | elapsed time per iteration (s): 5.34 | learning rate: 9.980E-04 | global batch size: 512 | lm loss: 5.479102E+00 | loss scale: 4096.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.818 | TFLOPs: 72.37 | +[default7]: iteration 463/ 3814 | consumed samples: 237056 | consumed tokens: 485490688 | elapsed time per iteration (s): 5.37 | learning rate: 9.979E-04 | global batch size: 512 | lm loss: 5.485051E+00 | loss scale: 4096.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.329 | TFLOPs: 72.00 | +[default7]: iteration 464/ 3814 | consumed samples: 237568 | consumed tokens: 486539264 | elapsed time per iteration (s): 5.37 | learning rate: 9.979E-04 | global batch size: 512 | lm loss: 5.485148E+00 | loss scale: 4096.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.420 | TFLOPs: 72.07 | +[default7]: iteration 465/ 3814 | consumed samples: 238080 | consumed tokens: 487587840 | elapsed time per iteration (s): 5.35 | learning rate: 9.979E-04 | global batch size: 512 | lm loss: 5.492012E+00 | loss scale: 4096.0 | grad norm: 0.179 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.698 | TFLOPs: 72.28 | +[default7]: iteration 466/ 3814 | consumed samples: 238592 | consumed tokens: 488636416 | elapsed time per iteration (s): 5.35 | learning rate: 9.978E-04 | global batch size: 512 | lm loss: 5.520459E+00 | loss scale: 4096.0 | grad norm: 0.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.751 | TFLOPs: 72.32 | +[default7]: iteration 467/ 3814 | consumed samples: 239104 | consumed tokens: 489684992 | elapsed time per iteration (s): 5.34 | learning rate: 9.978E-04 | global batch size: 512 | lm loss: 5.475989E+00 | loss scale: 4096.0 | grad norm: 0.192 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.911 | TFLOPs: 72.44 | +[default7]: iteration 468/ 3814 | consumed samples: 239616 | consumed tokens: 490733568 | elapsed time per iteration (s): 5.35 | learning rate: 9.977E-04 | global batch size: 512 | lm loss: 5.480297E+00 | loss scale: 4096.0 | grad norm: 0.195 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.636 | TFLOPs: 72.23 | +[default7]: iteration 469/ 3814 | consumed samples: 240128 | consumed tokens: 491782144 | elapsed time per iteration (s): 5.36 | learning rate: 9.977E-04 | global batch size: 512 | lm loss: 5.442911E+00 | loss scale: 4096.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.447 | TFLOPs: 72.09 | +[default7]: iteration 470/ 3814 | consumed samples: 240640 | consumed tokens: 492830720 | elapsed time per iteration (s): 5.34 | learning rate: 9.977E-04 | global batch size: 512 | lm loss: 5.449051E+00 | loss scale: 4096.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.830 | TFLOPs: 72.38 | +[default7]: iteration 471/ 3814 | consumed samples: 241152 | consumed tokens: 493879296 | elapsed time per iteration (s): 5.37 | learning rate: 9.976E-04 | global batch size: 512 | lm loss: 5.461061E+00 | loss scale: 4096.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.369 | TFLOPs: 72.03 | +[default7]: iteration 472/ 3814 | consumed samples: 241664 | consumed tokens: 494927872 | elapsed time per iteration (s): 5.38 | learning rate: 9.976E-04 | global batch size: 512 | lm loss: 5.427217E+00 | loss scale: 4096.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.183 | TFLOPs: 71.89 | +[default7]: iteration 473/ 3814 | consumed samples: 242176 | consumed tokens: 495976448 | elapsed time per iteration (s): 5.37 | learning rate: 9.975E-04 | global batch size: 512 | lm loss: 5.453721E+00 | loss scale: 4096.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.391 | TFLOPs: 72.05 | +[default7]: iteration 474/ 3814 | consumed samples: 242688 | consumed tokens: 497025024 | elapsed time per iteration (s): 5.35 | learning rate: 9.975E-04 | global batch size: 512 | lm loss: 5.445547E+00 | loss scale: 4096.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.759 | TFLOPs: 72.33 | +[default7]: iteration 475/ 3814 | consumed samples: 243200 | consumed tokens: 498073600 | elapsed time per iteration (s): 5.36 | learning rate: 9.974E-04 | global batch size: 512 | lm loss: 5.428763E+00 | loss scale: 4096.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.479 | TFLOPs: 72.12 | +[default7]: iteration 476/ 3814 | consumed samples: 243712 | consumed tokens: 499122176 | elapsed time per iteration (s): 5.35 | learning rate: 9.974E-04 | global batch size: 512 | lm loss: 5.441656E+00 | loss scale: 4096.0 | grad norm: 0.170 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.617 | TFLOPs: 72.22 | +[default7]: iteration 477/ 3814 | consumed samples: 244224 | consumed tokens: 500170752 | elapsed time per iteration (s): 5.36 | learning rate: 9.974E-04 | global batch size: 512 | lm loss: 5.448591E+00 | loss scale: 4096.0 | grad norm: 0.235 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.552 | TFLOPs: 72.17 | +[default7]: iteration 478/ 3814 | consumed samples: 244736 | consumed tokens: 501219328 | elapsed time per iteration (s): 5.37 | learning rate: 9.973E-04 | global batch size: 512 | lm loss: 5.446157E+00 | loss scale: 4096.0 | grad norm: 0.262 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.348 | TFLOPs: 72.02 | +[default7]: iteration 479/ 3814 | consumed samples: 245248 | consumed tokens: 502267904 | elapsed time per iteration (s): 5.37 | learning rate: 9.973E-04 | global batch size: 512 | lm loss: 5.427497E+00 | loss scale: 4096.0 | grad norm: 0.217 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.294 | TFLOPs: 71.98 | +[default7]: iteration 480/ 3814 | consumed samples: 245760 | consumed tokens: 503316480 | elapsed time per iteration (s): 5.37 | learning rate: 9.972E-04 | global batch size: 512 | lm loss: 5.455641E+00 | loss scale: 4096.0 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.316 | TFLOPs: 71.99 | +[default7]: iteration 481/ 3814 | consumed samples: 246272 | consumed tokens: 504365056 | elapsed time per iteration (s): 5.37 | learning rate: 9.972E-04 | global batch size: 512 | lm loss: 5.457831E+00 | loss scale: 4096.0 | grad norm: 0.220 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.353 | TFLOPs: 72.02 | +[default7]: iteration 482/ 3814 | consumed samples: 246784 | consumed tokens: 505413632 | elapsed time per iteration (s): 5.38 | learning rate: 9.971E-04 | global batch size: 512 | lm loss: 5.432716E+00 | loss scale: 4096.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.143 | TFLOPs: 71.86 | +[default7]: iteration 483/ 3814 | consumed samples: 247296 | consumed tokens: 506462208 | elapsed time per iteration (s): 5.37 | learning rate: 9.971E-04 | global batch size: 512 | lm loss: 5.420031E+00 | loss scale: 4096.0 | grad norm: 0.171 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.420 | TFLOPs: 72.07 | +[default7]: iteration 484/ 3814 | consumed samples: 247808 | consumed tokens: 507510784 | elapsed time per iteration (s): 5.38 | learning rate: 9.970E-04 | global batch size: 512 | lm loss: 5.422283E+00 | loss scale: 4096.0 | grad norm: 0.175 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.230 | TFLOPs: 71.93 | +[default7]: iteration 485/ 3814 | consumed samples: 248320 | consumed tokens: 508559360 | elapsed time per iteration (s): 5.39 | learning rate: 9.970E-04 | global batch size: 512 | lm loss: 5.414587E+00 | loss scale: 4096.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.016 | TFLOPs: 71.77 | +[default7]: iteration 486/ 3814 | consumed samples: 248832 | consumed tokens: 509607936 | elapsed time per iteration (s): 5.38 | learning rate: 9.969E-04 | global batch size: 512 | lm loss: 5.421704E+00 | loss scale: 4096.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.234 | TFLOPs: 71.93 | +[default7]: iteration 487/ 3814 | consumed samples: 249344 | consumed tokens: 510656512 | elapsed time per iteration (s): 5.36 | learning rate: 9.969E-04 | global batch size: 512 | lm loss: 5.412054E+00 | loss scale: 4096.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.507 | TFLOPs: 72.14 | +[default7]: iteration 488/ 3814 | consumed samples: 249856 | consumed tokens: 511705088 | elapsed time per iteration (s): 5.35 | learning rate: 9.968E-04 | global batch size: 512 | lm loss: 5.405315E+00 | loss scale: 4096.0 | grad norm: 0.197 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.646 | TFLOPs: 72.24 | +[default7]: iteration 489/ 3814 | consumed samples: 250368 | consumed tokens: 512753664 | elapsed time per iteration (s): 5.36 | learning rate: 9.968E-04 | global batch size: 512 | lm loss: 5.419230E+00 | loss scale: 4096.0 | grad norm: 0.242 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.480 | TFLOPs: 72.12 | +[default7]: iteration 490/ 3814 | consumed samples: 250880 | consumed tokens: 513802240 | elapsed time per iteration (s): 5.37 | learning rate: 9.967E-04 | global batch size: 512 | lm loss: 5.403038E+00 | loss scale: 4096.0 | grad norm: 0.194 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.408 | TFLOPs: 72.06 | +[default7]: iteration 491/ 3814 | consumed samples: 251392 | consumed tokens: 514850816 | elapsed time per iteration (s): 5.35 | learning rate: 9.967E-04 | global batch size: 512 | lm loss: 5.370651E+00 | loss scale: 4096.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.688 | TFLOPs: 72.27 | +[default7]: iteration 492/ 3814 | consumed samples: 251904 | consumed tokens: 515899392 | elapsed time per iteration (s): 5.35 | learning rate: 9.966E-04 | global batch size: 512 | lm loss: 5.377913E+00 | loss scale: 4096.0 | grad norm: 0.147 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.780 | TFLOPs: 72.34 | +[default7]: iteration 493/ 3814 | consumed samples: 252416 | consumed tokens: 516947968 | elapsed time per iteration (s): 5.39 | learning rate: 9.966E-04 | global batch size: 512 | lm loss: 5.386416E+00 | loss scale: 4096.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.061 | TFLOPs: 71.80 | +[default7]: iteration 494/ 3814 | consumed samples: 252928 | consumed tokens: 517996544 | elapsed time per iteration (s): 5.36 | learning rate: 9.965E-04 | global batch size: 512 | lm loss: 5.377167E+00 | loss scale: 4096.0 | grad norm: 0.215 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.498 | TFLOPs: 72.13 | +[default7]: iteration 495/ 3814 | consumed samples: 253440 | consumed tokens: 519045120 | elapsed time per iteration (s): 5.38 | learning rate: 9.965E-04 | global batch size: 512 | lm loss: 5.370601E+00 | loss scale: 4096.0 | grad norm: 0.191 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.205 | TFLOPs: 71.91 | +[default7]: iteration 496/ 3814 | consumed samples: 253952 | consumed tokens: 520093696 | elapsed time per iteration (s): 5.36 | learning rate: 9.964E-04 | global batch size: 512 | lm loss: 5.360343E+00 | loss scale: 4096.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.435 | TFLOPs: 72.08 | +[default7]: iteration 497/ 3814 | consumed samples: 254464 | consumed tokens: 521142272 | elapsed time per iteration (s): 5.37 | learning rate: 9.964E-04 | global batch size: 512 | lm loss: 5.361321E+00 | loss scale: 4096.0 | grad norm: 0.219 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.353 | TFLOPs: 72.02 | +[default7]: iteration 498/ 3814 | consumed samples: 254976 | consumed tokens: 522190848 | elapsed time per iteration (s): 5.36 | learning rate: 9.963E-04 | global batch size: 512 | lm loss: 5.360221E+00 | loss scale: 4096.0 | grad norm: 0.200 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.482 | TFLOPs: 72.12 | +[default7]: iteration 499/ 3814 | consumed samples: 255488 | consumed tokens: 523239424 | elapsed time per iteration (s): 5.36 | learning rate: 9.963E-04 | global batch size: 512 | lm loss: 5.361185E+00 | loss scale: 4096.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.456 | TFLOPs: 72.10 | +[default0]:saving checkpoint at iteration 500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 07:04:54,816] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step500 is begin to save! +[default0]:[2023-02-16 07:04:54,820] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:54,820] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_01-model_01-model_states.pt... +[default7]: iteration 500/ 3814 | consumed samples: 256000 | consumed tokens: 524288000 | elapsed time per iteration (s): 5.35 | learning rate: 9.962E-04 | global batch size: 512 | lm loss: 5.362911E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.716 | TFLOPs: 72.30 | +[default0]:[2023-02-16 07:04:54,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:54,982] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_04-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,041] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:54,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:54,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_04-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,039] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,095] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,150] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,092] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,143] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,203] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,254] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,255] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,259] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,260] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_09-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,306] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,367] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,368] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,381] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,381] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_11-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,437] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,437] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,425] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,496] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,552] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,552] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,475] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_13-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,529] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,530] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,608] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,609] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,584] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,585] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,664] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_16-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,724] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,699] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,762] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,801] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:55,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,854] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,823] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_19-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,911] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,876] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_20-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:55,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,928] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_21-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:55,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:55,964] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_21-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,016] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:55,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:55,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_22-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,033] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,071] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_23-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,132] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,133] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,094] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,095] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,156] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,200] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,213] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,262] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_26-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,315] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_27-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,325] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,326] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,370] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,370] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_28-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,426] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,379] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,379] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,433] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,434] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,481] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,482] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,536] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,490] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,544] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,593] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,594] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,648] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,648] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,598] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,599] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_33-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,650] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,651] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 07:04:56,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,707] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 07:04:56,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,708] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 07:04:56,771] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 07:04:56,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_37-model_01-model_states.pt. +[default0]:[2023-02-16 07:04:56,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,771] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,822] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 07:04:56,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 07:04:56,824] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_00_model_states.pt +[default0]:[2023-02-16 07:04:56,824] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 07:04:56,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_00_model_states.pt. +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:04:56,773] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_01_model_states.pt +[default1]:[2023-02-16 07:04:56,773] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 07:04:56,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:04:56,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:04:57,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,350] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,350] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,385] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,385] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,385] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,416] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,416] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,395] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,395] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,405] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,405] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,407] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,407] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,396] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,396] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,366] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,366] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,406] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,406] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,385] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,385] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,449] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,449] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,448] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,448] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,448] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,375] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,375] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,440] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,440] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,440] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,377] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,378] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,378] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,427] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,427] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,393] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,393] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,393] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,392] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,392] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,467] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,467] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,411] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,411] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,423] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,423] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,423] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,463] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,463] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,444] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,444] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,444] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,470] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,470] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,471] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,470] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,470] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,399] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,399] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,488] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,488] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,506] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,506] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,506] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,493] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,493] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,493] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,523] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,523] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,438] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,439] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,451] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,451] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,489] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,489] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,504] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,505] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,512] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,512] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,512] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,495] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,495] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,495] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,486] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,486] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,471] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,471] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,476] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,476] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,477] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,542] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,542] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,483] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,483] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,505] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,505] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,515] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,515] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,544] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,544] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,544] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,545] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,546] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,546] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,521] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,521] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,521] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,509] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,509] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,502] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,503] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,514] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,514] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,548] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,548] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,548] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,553] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,553] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,553] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,525] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,525] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,525] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,514] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,514] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,518] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,518] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,518] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default1]:[2023-02-16 07:04:57,573] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:04:57,573] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:04:57,573] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default4]:[2023-02-16 07:04:57,558] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:04:57,559] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:04:57,559] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,580] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,580] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default2]:[2023-02-16 07:04:57,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:04:57,547] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:04:57,547] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:[2023-02-16 07:04:57,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,565] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,565] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default6]:[2023-02-16 07:04:57,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:04:57,540] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:04:57,540] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default3]:[2023-02-16 07:04:57,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:04:57,637] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:04:57,637] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]:[2023-02-16 07:04:57,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:04:57,704] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:04:57,705] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default5]:[2023-02-16 07:04:57,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:04:57,736] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:04:57,736] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default0]: successfully saved checkpoint at iteration 500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]:[2023-02-16 07:04:57,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:04:57,710] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step500/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:04:57,710] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step500 is ready now! +[default7]:time (ms) | save-checkpoint: 2921.56 +[default7]: iteration 501/ 3814 | consumed samples: 256512 | consumed tokens: 525336576 | elapsed time per iteration (s): 8.29 | learning rate: 9.962E-04 | global batch size: 512 | lm loss: 5.342995E+00 | loss scale: 8192.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.792 | TFLOPs: 46.67 | +[default7]: iteration 502/ 3814 | consumed samples: 257024 | consumed tokens: 526385152 | elapsed time per iteration (s): 5.38 | learning rate: 9.961E-04 | global batch size: 512 | lm loss: 5.353068E+00 | loss scale: 8192.0 | grad norm: 0.171 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.220 | TFLOPs: 71.92 | +[default7]: iteration 503/ 3814 | consumed samples: 257536 | consumed tokens: 527433728 | elapsed time per iteration (s): 5.39 | learning rate: 9.961E-04 | global batch size: 512 | lm loss: 5.332346E+00 | loss scale: 8192.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.031 | TFLOPs: 71.78 | +[default7]: iteration 504/ 3814 | consumed samples: 258048 | consumed tokens: 528482304 | elapsed time per iteration (s): 5.37 | learning rate: 9.960E-04 | global batch size: 512 | lm loss: 5.351198E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.282 | TFLOPs: 71.97 | +[default7]: iteration 505/ 3814 | consumed samples: 258560 | consumed tokens: 529530880 | elapsed time per iteration (s): 5.36 | learning rate: 9.960E-04 | global batch size: 512 | lm loss: 5.320243E+00 | loss scale: 8192.0 | grad norm: 0.191 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.443 | TFLOPs: 72.09 | +[default7]: iteration 506/ 3814 | consumed samples: 259072 | consumed tokens: 530579456 | elapsed time per iteration (s): 5.37 | learning rate: 9.959E-04 | global batch size: 512 | lm loss: 5.340350E+00 | loss scale: 8192.0 | grad norm: 0.222 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.360 | TFLOPs: 72.03 | +[default7]: iteration 507/ 3814 | consumed samples: 259584 | consumed tokens: 531628032 | elapsed time per iteration (s): 5.37 | learning rate: 9.959E-04 | global batch size: 512 | lm loss: 5.338311E+00 | loss scale: 8192.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.263 | TFLOPs: 71.95 | +[default7]: iteration 508/ 3814 | consumed samples: 260096 | consumed tokens: 532676608 | elapsed time per iteration (s): 5.37 | learning rate: 9.958E-04 | global batch size: 512 | lm loss: 5.337334E+00 | loss scale: 8192.0 | grad norm: 0.151 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.336 | TFLOPs: 72.01 | +[default7]: iteration 509/ 3814 | consumed samples: 260608 | consumed tokens: 533725184 | elapsed time per iteration (s): 5.37 | learning rate: 9.957E-04 | global batch size: 512 | lm loss: 5.341227E+00 | loss scale: 8192.0 | grad norm: 0.151 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.301 | TFLOPs: 71.98 | +[default7]: iteration 510/ 3814 | consumed samples: 261120 | consumed tokens: 534773760 | elapsed time per iteration (s): 5.37 | learning rate: 9.957E-04 | global batch size: 512 | lm loss: 5.322726E+00 | loss scale: 8192.0 | grad norm: 0.175 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.380 | TFLOPs: 72.04 | +[default7]: iteration 511/ 3814 | consumed samples: 261632 | consumed tokens: 535822336 | elapsed time per iteration (s): 5.39 | learning rate: 9.956E-04 | global batch size: 512 | lm loss: 5.333968E+00 | loss scale: 8192.0 | grad norm: 0.238 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.059 | TFLOPs: 71.80 | +[default7]: iteration 512/ 3814 | consumed samples: 262144 | consumed tokens: 536870912 | elapsed time per iteration (s): 5.37 | learning rate: 9.956E-04 | global batch size: 512 | lm loss: 5.329570E+00 | loss scale: 8192.0 | grad norm: 0.251 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.325 | TFLOPs: 72.00 | +[default7]: iteration 513/ 3814 | consumed samples: 262656 | consumed tokens: 537919488 | elapsed time per iteration (s): 5.39 | learning rate: 9.955E-04 | global batch size: 512 | lm loss: 5.290391E+00 | loss scale: 8192.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.032 | TFLOPs: 71.78 | +[default7]: iteration 514/ 3814 | consumed samples: 263168 | consumed tokens: 538968064 | elapsed time per iteration (s): 5.39 | learning rate: 9.955E-04 | global batch size: 512 | lm loss: 5.302201E+00 | loss scale: 8192.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.064 | TFLOPs: 71.80 | +[default7]: iteration 515/ 3814 | consumed samples: 263680 | consumed tokens: 540016640 | elapsed time per iteration (s): 5.37 | learning rate: 9.954E-04 | global batch size: 512 | lm loss: 5.275203E+00 | loss scale: 8192.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.395 | TFLOPs: 72.05 | +[default7]: iteration 516/ 3814 | consumed samples: 264192 | consumed tokens: 541065216 | elapsed time per iteration (s): 5.40 | learning rate: 9.953E-04 | global batch size: 512 | lm loss: 5.253895E+00 | loss scale: 8192.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.858 | TFLOPs: 71.65 | +[default7]: iteration 517/ 3814 | consumed samples: 264704 | consumed tokens: 542113792 | elapsed time per iteration (s): 5.37 | learning rate: 9.953E-04 | global batch size: 512 | lm loss: 5.267434E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.432 | TFLOPs: 72.08 | +[default7]: iteration 518/ 3814 | consumed samples: 265216 | consumed tokens: 543162368 | elapsed time per iteration (s): 5.38 | learning rate: 9.952E-04 | global batch size: 512 | lm loss: 5.263441E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.175 | TFLOPs: 71.89 | +[default7]: iteration 519/ 3814 | consumed samples: 265728 | consumed tokens: 544210944 | elapsed time per iteration (s): 5.39 | learning rate: 9.952E-04 | global batch size: 512 | lm loss: 5.268166E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.998 | TFLOPs: 71.75 | +[default7]: iteration 520/ 3814 | consumed samples: 266240 | consumed tokens: 545259520 | elapsed time per iteration (s): 5.37 | learning rate: 9.951E-04 | global batch size: 512 | lm loss: 5.252014E+00 | loss scale: 8192.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.372 | TFLOPs: 72.04 | +[default7]: iteration 521/ 3814 | consumed samples: 266752 | consumed tokens: 546308096 | elapsed time per iteration (s): 5.37 | learning rate: 9.950E-04 | global batch size: 512 | lm loss: 5.224272E+00 | loss scale: 8192.0 | grad norm: 0.168 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.381 | TFLOPs: 72.04 | +[default7]: iteration 522/ 3814 | consumed samples: 267264 | consumed tokens: 547356672 | elapsed time per iteration (s): 5.37 | learning rate: 9.950E-04 | global batch size: 512 | lm loss: 5.261168E+00 | loss scale: 8192.0 | grad norm: 0.246 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.266 | TFLOPs: 71.96 | +[default7]: iteration 523/ 3814 | consumed samples: 267776 | consumed tokens: 548405248 | elapsed time per iteration (s): 5.39 | learning rate: 9.949E-04 | global batch size: 512 | lm loss: 5.256585E+00 | loss scale: 8192.0 | grad norm: 0.187 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.973 | TFLOPs: 71.73 | +[default7]: iteration 524/ 3814 | consumed samples: 268288 | consumed tokens: 549453824 | elapsed time per iteration (s): 5.38 | learning rate: 9.949E-04 | global batch size: 512 | lm loss: 5.244722E+00 | loss scale: 8192.0 | grad norm: 0.195 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.230 | TFLOPs: 71.93 | +[default7]: iteration 525/ 3814 | consumed samples: 268800 | consumed tokens: 550502400 | elapsed time per iteration (s): 5.38 | learning rate: 9.948E-04 | global batch size: 512 | lm loss: 5.287830E+00 | loss scale: 8192.0 | grad norm: 0.186 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.108 | TFLOPs: 71.84 | +[default7]: iteration 526/ 3814 | consumed samples: 269312 | consumed tokens: 551550976 | elapsed time per iteration (s): 5.41 | learning rate: 9.947E-04 | global batch size: 512 | lm loss: 5.270915E+00 | loss scale: 8192.0 | grad norm: 0.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.723 | TFLOPs: 71.55 | +[default7]: iteration 527/ 3814 | consumed samples: 269824 | consumed tokens: 552599552 | elapsed time per iteration (s): 5.40 | learning rate: 9.947E-04 | global batch size: 512 | lm loss: 5.253179E+00 | loss scale: 8192.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.843 | TFLOPs: 71.64 | +[default7]: iteration 528/ 3814 | consumed samples: 270336 | consumed tokens: 553648128 | elapsed time per iteration (s): 5.37 | learning rate: 9.946E-04 | global batch size: 512 | lm loss: 5.258872E+00 | loss scale: 8192.0 | grad norm: 0.166 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.366 | TFLOPs: 72.03 | +[default7]: iteration 529/ 3814 | consumed samples: 270848 | consumed tokens: 554696704 | elapsed time per iteration (s): 5.38 | learning rate: 9.946E-04 | global batch size: 512 | lm loss: 5.272453E+00 | loss scale: 8192.0 | grad norm: 0.200 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.214 | TFLOPs: 71.92 | +[default7]: iteration 530/ 3814 | consumed samples: 271360 | consumed tokens: 555745280 | elapsed time per iteration (s): 5.37 | learning rate: 9.945E-04 | global batch size: 512 | lm loss: 5.238136E+00 | loss scale: 8192.0 | grad norm: 0.179 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.300 | TFLOPs: 71.98 | +[default7]: iteration 531/ 3814 | consumed samples: 271872 | consumed tokens: 556793856 | elapsed time per iteration (s): 5.38 | learning rate: 9.944E-04 | global batch size: 512 | lm loss: 5.248231E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.233 | TFLOPs: 71.93 | +[default7]: iteration 532/ 3814 | consumed samples: 272384 | consumed tokens: 557842432 | elapsed time per iteration (s): 5.38 | learning rate: 9.944E-04 | global batch size: 512 | lm loss: 5.221687E+00 | loss scale: 8192.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.134 | TFLOPs: 71.86 | +[default7]: iteration 533/ 3814 | consumed samples: 272896 | consumed tokens: 558891008 | elapsed time per iteration (s): 5.38 | learning rate: 9.943E-04 | global batch size: 512 | lm loss: 5.208799E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.201 | TFLOPs: 71.91 | +[default7]: iteration 534/ 3814 | consumed samples: 273408 | consumed tokens: 559939584 | elapsed time per iteration (s): 5.36 | learning rate: 9.942E-04 | global batch size: 512 | lm loss: 5.191990E+00 | loss scale: 8192.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.485 | TFLOPs: 72.12 | +[default7]: iteration 535/ 3814 | consumed samples: 273920 | consumed tokens: 560988160 | elapsed time per iteration (s): 5.36 | learning rate: 9.942E-04 | global batch size: 512 | lm loss: 5.203464E+00 | loss scale: 8192.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.578 | TFLOPs: 72.19 | +[default7]: iteration 536/ 3814 | consumed samples: 274432 | consumed tokens: 562036736 | elapsed time per iteration (s): 5.37 | learning rate: 9.941E-04 | global batch size: 512 | lm loss: 5.208980E+00 | loss scale: 8192.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.279 | TFLOPs: 71.96 | +[default7]: iteration 537/ 3814 | consumed samples: 274944 | consumed tokens: 563085312 | elapsed time per iteration (s): 5.36 | learning rate: 9.940E-04 | global batch size: 512 | lm loss: 5.207233E+00 | loss scale: 8192.0 | grad norm: 0.179 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.547 | TFLOPs: 72.17 | +[default7]: iteration 538/ 3814 | consumed samples: 275456 | consumed tokens: 564133888 | elapsed time per iteration (s): 5.38 | learning rate: 9.940E-04 | global batch size: 512 | lm loss: 5.208960E+00 | loss scale: 8192.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.224 | TFLOPs: 71.92 | +[default7]: iteration 539/ 3814 | consumed samples: 275968 | consumed tokens: 565182464 | elapsed time per iteration (s): 5.37 | learning rate: 9.939E-04 | global batch size: 512 | lm loss: 5.209212E+00 | loss scale: 8192.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.352 | TFLOPs: 72.02 | +[default7]: iteration 540/ 3814 | consumed samples: 276480 | consumed tokens: 566231040 | elapsed time per iteration (s): 5.36 | learning rate: 9.938E-04 | global batch size: 512 | lm loss: 5.199387E+00 | loss scale: 8192.0 | grad norm: 0.144 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.516 | TFLOPs: 72.14 | +[default7]: iteration 541/ 3814 | consumed samples: 276992 | consumed tokens: 567279616 | elapsed time per iteration (s): 5.37 | learning rate: 9.938E-04 | global batch size: 512 | lm loss: 5.194688E+00 | loss scale: 8192.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.409 | TFLOPs: 72.06 | +[default7]: iteration 542/ 3814 | consumed samples: 277504 | consumed tokens: 568328192 | elapsed time per iteration (s): 5.38 | learning rate: 9.937E-04 | global batch size: 512 | lm loss: 5.163635E+00 | loss scale: 8192.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.217 | TFLOPs: 71.92 | +[default7]: iteration 543/ 3814 | consumed samples: 278016 | consumed tokens: 569376768 | elapsed time per iteration (s): 5.36 | learning rate: 9.936E-04 | global batch size: 512 | lm loss: 5.171818E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.544 | TFLOPs: 72.17 | +[default7]: iteration 544/ 3814 | consumed samples: 278528 | consumed tokens: 570425344 | elapsed time per iteration (s): 5.39 | learning rate: 9.936E-04 | global batch size: 512 | lm loss: 5.202235E+00 | loss scale: 8192.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.026 | TFLOPs: 71.77 | +[default7]: iteration 545/ 3814 | consumed samples: 279040 | consumed tokens: 571473920 | elapsed time per iteration (s): 5.39 | learning rate: 9.935E-04 | global batch size: 512 | lm loss: 5.162260E+00 | loss scale: 8192.0 | grad norm: 0.190 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.938 | TFLOPs: 71.71 | +[default7]: iteration 546/ 3814 | consumed samples: 279552 | consumed tokens: 572522496 | elapsed time per iteration (s): 5.40 | learning rate: 9.934E-04 | global batch size: 512 | lm loss: 5.171312E+00 | loss scale: 8192.0 | grad norm: 0.186 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.857 | TFLOPs: 71.65 | +[default7]: iteration 547/ 3814 | consumed samples: 280064 | consumed tokens: 573571072 | elapsed time per iteration (s): 5.41 | learning rate: 9.934E-04 | global batch size: 512 | lm loss: 5.180484E+00 | loss scale: 8192.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.669 | TFLOPs: 71.50 | +[default7]: iteration 548/ 3814 | consumed samples: 280576 | consumed tokens: 574619648 | elapsed time per iteration (s): 5.38 | learning rate: 9.933E-04 | global batch size: 512 | lm loss: 5.136375E+00 | loss scale: 8192.0 | grad norm: 0.168 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.237 | TFLOPs: 71.93 | +[default7]: iteration 549/ 3814 | consumed samples: 281088 | consumed tokens: 575668224 | elapsed time per iteration (s): 5.36 | learning rate: 9.932E-04 | global batch size: 512 | lm loss: 5.158598E+00 | loss scale: 8192.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.449 | TFLOPs: 72.09 | +[default7]: iteration 550/ 3814 | consumed samples: 281600 | consumed tokens: 576716800 | elapsed time per iteration (s): 5.40 | learning rate: 9.931E-04 | global batch size: 512 | lm loss: 5.128084E+00 | loss scale: 8192.0 | grad norm: 0.171 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.841 | TFLOPs: 71.63 | +[default7]: iteration 551/ 3814 | consumed samples: 282112 | consumed tokens: 577765376 | elapsed time per iteration (s): 5.39 | learning rate: 9.931E-04 | global batch size: 512 | lm loss: 5.141730E+00 | loss scale: 8192.0 | grad norm: 0.159 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.909 | TFLOPs: 71.69 | +[default7]: iteration 552/ 3814 | consumed samples: 282624 | consumed tokens: 578813952 | elapsed time per iteration (s): 5.39 | learning rate: 9.930E-04 | global batch size: 512 | lm loss: 5.114822E+00 | loss scale: 8192.0 | grad norm: 0.170 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.907 | TFLOPs: 71.68 | +[default7]: iteration 553/ 3814 | consumed samples: 283136 | consumed tokens: 579862528 | elapsed time per iteration (s): 5.38 | learning rate: 9.929E-04 | global batch size: 512 | lm loss: 5.139152E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.087 | TFLOPs: 71.82 | +[default7]: iteration 554/ 3814 | consumed samples: 283648 | consumed tokens: 580911104 | elapsed time per iteration (s): 5.37 | learning rate: 9.929E-04 | global batch size: 512 | lm loss: 5.109661E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.305 | TFLOPs: 71.98 | +[default7]: iteration 555/ 3814 | consumed samples: 284160 | consumed tokens: 581959680 | elapsed time per iteration (s): 5.36 | learning rate: 9.928E-04 | global batch size: 512 | lm loss: 5.080564E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.536 | TFLOPs: 72.16 | +[default7]: iteration 556/ 3814 | consumed samples: 284672 | consumed tokens: 583008256 | elapsed time per iteration (s): 5.36 | learning rate: 9.927E-04 | global batch size: 512 | lm loss: 5.104445E+00 | loss scale: 8192.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.526 | TFLOPs: 72.15 | +[default7]: iteration 557/ 3814 | consumed samples: 285184 | consumed tokens: 584056832 | elapsed time per iteration (s): 5.40 | learning rate: 9.926E-04 | global batch size: 512 | lm loss: 5.106403E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.868 | TFLOPs: 71.65 | +[default7]: iteration 558/ 3814 | consumed samples: 285696 | consumed tokens: 585105408 | elapsed time per iteration (s): 5.38 | learning rate: 9.926E-04 | global batch size: 512 | lm loss: 5.131998E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.207 | TFLOPs: 71.91 | +[default7]: iteration 559/ 3814 | consumed samples: 286208 | consumed tokens: 586153984 | elapsed time per iteration (s): 5.38 | learning rate: 9.925E-04 | global batch size: 512 | lm loss: 5.099446E+00 | loss scale: 8192.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.188 | TFLOPs: 71.90 | +[default7]: iteration 560/ 3814 | consumed samples: 286720 | consumed tokens: 587202560 | elapsed time per iteration (s): 5.39 | learning rate: 9.924E-04 | global batch size: 512 | lm loss: 5.083074E+00 | loss scale: 8192.0 | grad norm: 0.202 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.051 | TFLOPs: 71.79 | +[default7]: iteration 561/ 3814 | consumed samples: 287232 | consumed tokens: 588251136 | elapsed time per iteration (s): 5.39 | learning rate: 9.923E-04 | global batch size: 512 | lm loss: 5.125063E+00 | loss scale: 8192.0 | grad norm: 0.213 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.909 | TFLOPs: 71.69 | +[default7]: iteration 562/ 3814 | consumed samples: 287744 | consumed tokens: 589299712 | elapsed time per iteration (s): 5.38 | learning rate: 9.923E-04 | global batch size: 512 | lm loss: 5.110170E+00 | loss scale: 8192.0 | grad norm: 0.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.118 | TFLOPs: 71.84 | +[default7]: iteration 563/ 3814 | consumed samples: 288256 | consumed tokens: 590348288 | elapsed time per iteration (s): 5.38 | learning rate: 9.922E-04 | global batch size: 512 | lm loss: 5.096154E+00 | loss scale: 8192.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.091 | TFLOPs: 71.82 | +[default7]: iteration 564/ 3814 | consumed samples: 288768 | consumed tokens: 591396864 | elapsed time per iteration (s): 5.39 | learning rate: 9.921E-04 | global batch size: 512 | lm loss: 5.100950E+00 | loss scale: 8192.0 | grad norm: 0.162 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.008 | TFLOPs: 71.76 | +[default7]: iteration 565/ 3814 | consumed samples: 289280 | consumed tokens: 592445440 | elapsed time per iteration (s): 5.39 | learning rate: 9.920E-04 | global batch size: 512 | lm loss: 5.107890E+00 | loss scale: 8192.0 | grad norm: 0.172 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.925 | TFLOPs: 71.70 | +[default7]: iteration 566/ 3814 | consumed samples: 289792 | consumed tokens: 593494016 | elapsed time per iteration (s): 5.37 | learning rate: 9.920E-04 | global batch size: 512 | lm loss: 5.082269E+00 | loss scale: 8192.0 | grad norm: 0.159 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.433 | TFLOPs: 72.08 | +[default7]: iteration 567/ 3814 | consumed samples: 290304 | consumed tokens: 594542592 | elapsed time per iteration (s): 5.37 | learning rate: 9.919E-04 | global batch size: 512 | lm loss: 5.068576E+00 | loss scale: 8192.0 | grad norm: 0.183 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.322 | TFLOPs: 72.00 | +[default7]: iteration 568/ 3814 | consumed samples: 290816 | consumed tokens: 595591168 | elapsed time per iteration (s): 5.37 | learning rate: 9.918E-04 | global batch size: 512 | lm loss: 5.071043E+00 | loss scale: 8192.0 | grad norm: 0.177 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.298 | TFLOPs: 71.98 | +[default7]: iteration 569/ 3814 | consumed samples: 291328 | consumed tokens: 596639744 | elapsed time per iteration (s): 5.38 | learning rate: 9.917E-04 | global batch size: 512 | lm loss: 5.080711E+00 | loss scale: 8192.0 | grad norm: 0.166 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.126 | TFLOPs: 71.85 | +[default7]: iteration 570/ 3814 | consumed samples: 291840 | consumed tokens: 597688320 | elapsed time per iteration (s): 5.36 | learning rate: 9.916E-04 | global batch size: 512 | lm loss: 5.090583E+00 | loss scale: 8192.0 | grad norm: 0.181 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.500 | TFLOPs: 72.13 | +[default7]: iteration 571/ 3814 | consumed samples: 292352 | consumed tokens: 598736896 | elapsed time per iteration (s): 5.39 | learning rate: 9.916E-04 | global batch size: 512 | lm loss: 5.097588E+00 | loss scale: 8192.0 | grad norm: 0.167 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.930 | TFLOPs: 71.70 | +[default7]: iteration 572/ 3814 | consumed samples: 292864 | consumed tokens: 599785472 | elapsed time per iteration (s): 5.38 | learning rate: 9.915E-04 | global batch size: 512 | lm loss: 5.087179E+00 | loss scale: 8192.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.250 | TFLOPs: 71.94 | +[default7]: iteration 573/ 3814 | consumed samples: 293376 | consumed tokens: 600834048 | elapsed time per iteration (s): 5.38 | learning rate: 9.914E-04 | global batch size: 512 | lm loss: 5.084613E+00 | loss scale: 8192.0 | grad norm: 0.214 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.219 | TFLOPs: 71.92 | +[default7]: iteration 574/ 3814 | consumed samples: 293888 | consumed tokens: 601882624 | elapsed time per iteration (s): 5.37 | learning rate: 9.913E-04 | global batch size: 512 | lm loss: 5.097101E+00 | loss scale: 8192.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.374 | TFLOPs: 72.04 | +[default7]: iteration 575/ 3814 | consumed samples: 294400 | consumed tokens: 602931200 | elapsed time per iteration (s): 5.37 | learning rate: 9.912E-04 | global batch size: 512 | lm loss: 5.069646E+00 | loss scale: 8192.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.328 | TFLOPs: 72.00 | +[default7]: iteration 576/ 3814 | consumed samples: 294912 | consumed tokens: 603979776 | elapsed time per iteration (s): 5.38 | learning rate: 9.912E-04 | global batch size: 512 | lm loss: 5.071320E+00 | loss scale: 8192.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.125 | TFLOPs: 71.85 | +[default7]: iteration 577/ 3814 | consumed samples: 295424 | consumed tokens: 605028352 | elapsed time per iteration (s): 5.38 | learning rate: 9.911E-04 | global batch size: 512 | lm loss: 5.040969E+00 | loss scale: 8192.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.219 | TFLOPs: 71.92 | +[default7]: iteration 578/ 3814 | consumed samples: 295936 | consumed tokens: 606076928 | elapsed time per iteration (s): 5.36 | learning rate: 9.910E-04 | global batch size: 512 | lm loss: 5.076215E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.485 | TFLOPs: 72.12 | +[default7]: iteration 579/ 3814 | consumed samples: 296448 | consumed tokens: 607125504 | elapsed time per iteration (s): 5.36 | learning rate: 9.909E-04 | global batch size: 512 | lm loss: 5.033697E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.491 | TFLOPs: 72.13 | +[default7]: iteration 580/ 3814 | consumed samples: 296960 | consumed tokens: 608174080 | elapsed time per iteration (s): 5.37 | learning rate: 9.908E-04 | global batch size: 512 | lm loss: 5.014784E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.361 | TFLOPs: 72.03 | +[default7]: iteration 581/ 3814 | consumed samples: 297472 | consumed tokens: 609222656 | elapsed time per iteration (s): 5.38 | learning rate: 9.908E-04 | global batch size: 512 | lm loss: 5.021296E+00 | loss scale: 8192.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.121 | TFLOPs: 71.85 | +[default7]: iteration 582/ 3814 | consumed samples: 297984 | consumed tokens: 610271232 | elapsed time per iteration (s): 5.36 | learning rate: 9.907E-04 | global batch size: 512 | lm loss: 5.033078E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.443 | TFLOPs: 72.09 | +[default7]: iteration 583/ 3814 | consumed samples: 298496 | consumed tokens: 611319808 | elapsed time per iteration (s): 5.35 | learning rate: 9.906E-04 | global batch size: 512 | lm loss: 5.007174E+00 | loss scale: 8192.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.666 | TFLOPs: 72.26 | +[default7]: iteration 584/ 3814 | consumed samples: 299008 | consumed tokens: 612368384 | elapsed time per iteration (s): 5.37 | learning rate: 9.905E-04 | global batch size: 512 | lm loss: 5.006625E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.392 | TFLOPs: 72.05 | +[default7]: iteration 585/ 3814 | consumed samples: 299520 | consumed tokens: 613416960 | elapsed time per iteration (s): 5.39 | learning rate: 9.904E-04 | global batch size: 512 | lm loss: 5.027624E+00 | loss scale: 8192.0 | grad norm: 0.159 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.078 | TFLOPs: 71.81 | +[default7]: iteration 586/ 3814 | consumed samples: 300032 | consumed tokens: 614465536 | elapsed time per iteration (s): 5.37 | learning rate: 9.903E-04 | global batch size: 512 | lm loss: 5.031887E+00 | loss scale: 8192.0 | grad norm: 0.182 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.307 | TFLOPs: 71.99 | +[default7]: iteration 587/ 3814 | consumed samples: 300544 | consumed tokens: 615514112 | elapsed time per iteration (s): 5.37 | learning rate: 9.903E-04 | global batch size: 512 | lm loss: 5.043273E+00 | loss scale: 8192.0 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.363 | TFLOPs: 72.03 | +[default7]: iteration 588/ 3814 | consumed samples: 301056 | consumed tokens: 616562688 | elapsed time per iteration (s): 5.35 | learning rate: 9.902E-04 | global batch size: 512 | lm loss: 5.038588E+00 | loss scale: 8192.0 | grad norm: 0.241 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.648 | TFLOPs: 72.24 | +[default7]: iteration 589/ 3814 | consumed samples: 301568 | consumed tokens: 617611264 | elapsed time per iteration (s): 5.38 | learning rate: 9.901E-04 | global batch size: 512 | lm loss: 5.030405E+00 | loss scale: 8192.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.193 | TFLOPs: 71.90 | +[default7]: iteration 590/ 3814 | consumed samples: 302080 | consumed tokens: 618659840 | elapsed time per iteration (s): 5.36 | learning rate: 9.900E-04 | global batch size: 512 | lm loss: 4.994034E+00 | loss scale: 8192.0 | grad norm: 0.166 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.450 | TFLOPs: 72.09 | +[default7]: iteration 591/ 3814 | consumed samples: 302592 | consumed tokens: 619708416 | elapsed time per iteration (s): 5.37 | learning rate: 9.899E-04 | global batch size: 512 | lm loss: 5.015311E+00 | loss scale: 8192.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.363 | TFLOPs: 72.03 | +[default7]: iteration 592/ 3814 | consumed samples: 303104 | consumed tokens: 620756992 | elapsed time per iteration (s): 5.36 | learning rate: 9.898E-04 | global batch size: 512 | lm loss: 4.988499E+00 | loss scale: 8192.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.536 | TFLOPs: 72.16 | +[default7]: iteration 593/ 3814 | consumed samples: 303616 | consumed tokens: 621805568 | elapsed time per iteration (s): 5.37 | learning rate: 9.897E-04 | global batch size: 512 | lm loss: 4.979034E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.382 | TFLOPs: 72.04 | +[default7]: iteration 594/ 3814 | consumed samples: 304128 | consumed tokens: 622854144 | elapsed time per iteration (s): 5.37 | learning rate: 9.897E-04 | global batch size: 512 | lm loss: 4.983533E+00 | loss scale: 8192.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.403 | TFLOPs: 72.06 | +[default7]: iteration 595/ 3814 | consumed samples: 304640 | consumed tokens: 623902720 | elapsed time per iteration (s): 5.35 | learning rate: 9.896E-04 | global batch size: 512 | lm loss: 4.986733E+00 | loss scale: 8192.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.703 | TFLOPs: 72.29 | +[default7]: iteration 596/ 3814 | consumed samples: 305152 | consumed tokens: 624951296 | elapsed time per iteration (s): 5.35 | learning rate: 9.895E-04 | global batch size: 512 | lm loss: 4.960186E+00 | loss scale: 8192.0 | grad norm: 0.163 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.777 | TFLOPs: 72.34 | +[default7]: iteration 597/ 3814 | consumed samples: 305664 | consumed tokens: 625999872 | elapsed time per iteration (s): 5.36 | learning rate: 9.894E-04 | global batch size: 512 | lm loss: 4.968796E+00 | loss scale: 8192.0 | grad norm: 0.165 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.556 | TFLOPs: 72.17 | +[default7]: iteration 598/ 3814 | consumed samples: 306176 | consumed tokens: 627048448 | elapsed time per iteration (s): 5.38 | learning rate: 9.893E-04 | global batch size: 512 | lm loss: 4.965558E+00 | loss scale: 8192.0 | grad norm: 0.172 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.172 | TFLOPs: 71.88 | +[default7]: iteration 599/ 3814 | consumed samples: 306688 | consumed tokens: 628097024 | elapsed time per iteration (s): 5.37 | learning rate: 9.892E-04 | global batch size: 512 | lm loss: 4.978924E+00 | loss scale: 8192.0 | grad norm: 0.164 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.345 | TFLOPs: 72.02 | +[default7]: iteration 600/ 3814 | consumed samples: 307200 | consumed tokens: 629145600 | elapsed time per iteration (s): 5.37 | learning rate: 9.891E-04 | global batch size: 512 | lm loss: 4.958943E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.401 | TFLOPs: 72.06 | +[default7]: iteration 601/ 3814 | consumed samples: 307712 | consumed tokens: 630194176 | elapsed time per iteration (s): 5.37 | learning rate: 9.890E-04 | global batch size: 512 | lm loss: 4.975012E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.263 | TFLOPs: 71.95 | +[default7]: iteration 602/ 3814 | consumed samples: 308224 | consumed tokens: 631242752 | elapsed time per iteration (s): 5.39 | learning rate: 9.889E-04 | global batch size: 512 | lm loss: 4.960578E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.041 | TFLOPs: 71.79 | +[default7]: iteration 603/ 3814 | consumed samples: 308736 | consumed tokens: 632291328 | elapsed time per iteration (s): 5.37 | learning rate: 9.889E-04 | global batch size: 512 | lm loss: 4.927587E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.346 | TFLOPs: 72.02 | +[default7]: iteration 604/ 3814 | consumed samples: 309248 | consumed tokens: 633339904 | elapsed time per iteration (s): 5.37 | learning rate: 9.888E-04 | global batch size: 512 | lm loss: 4.945430E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.304 | TFLOPs: 71.98 | +[default7]: iteration 605/ 3814 | consumed samples: 309760 | consumed tokens: 634388480 | elapsed time per iteration (s): 5.39 | learning rate: 9.887E-04 | global batch size: 512 | lm loss: 4.905688E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.916 | TFLOPs: 71.69 | +[default7]: iteration 606/ 3814 | consumed samples: 310272 | consumed tokens: 635437056 | elapsed time per iteration (s): 5.37 | learning rate: 9.886E-04 | global batch size: 512 | lm loss: 4.922870E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.413 | TFLOPs: 72.07 | +[default7]: iteration 607/ 3814 | consumed samples: 310784 | consumed tokens: 636485632 | elapsed time per iteration (s): 5.36 | learning rate: 9.885E-04 | global batch size: 512 | lm loss: 4.898771E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.440 | TFLOPs: 72.09 | +[default7]: iteration 608/ 3814 | consumed samples: 311296 | consumed tokens: 637534208 | elapsed time per iteration (s): 5.36 | learning rate: 9.884E-04 | global batch size: 512 | lm loss: 4.874371E+00 | loss scale: 8192.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.535 | TFLOPs: 72.16 | +[default7]: iteration 609/ 3814 | consumed samples: 311808 | consumed tokens: 638582784 | elapsed time per iteration (s): 5.38 | learning rate: 9.883E-04 | global batch size: 512 | lm loss: 4.928108E+00 | loss scale: 8192.0 | grad norm: 0.150 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.204 | TFLOPs: 71.91 | +[default7]: iteration 610/ 3814 | consumed samples: 312320 | consumed tokens: 639631360 | elapsed time per iteration (s): 5.40 | learning rate: 9.882E-04 | global batch size: 512 | lm loss: 4.920246E+00 | loss scale: 8192.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.779 | TFLOPs: 71.59 | +[default7]: iteration 611/ 3814 | consumed samples: 312832 | consumed tokens: 640679936 | elapsed time per iteration (s): 5.39 | learning rate: 9.881E-04 | global batch size: 512 | lm loss: 4.923007E+00 | loss scale: 8192.0 | grad norm: 0.184 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.971 | TFLOPs: 71.73 | +[default7]: iteration 612/ 3814 | consumed samples: 313344 | consumed tokens: 641728512 | elapsed time per iteration (s): 5.37 | learning rate: 9.880E-04 | global batch size: 512 | lm loss: 4.938094E+00 | loss scale: 8192.0 | grad norm: 0.181 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.405 | TFLOPs: 72.06 | +[default7]: iteration 613/ 3814 | consumed samples: 313856 | consumed tokens: 642777088 | elapsed time per iteration (s): 5.39 | learning rate: 9.879E-04 | global batch size: 512 | lm loss: 4.916399E+00 | loss scale: 8192.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.969 | TFLOPs: 71.73 | +[default7]: iteration 614/ 3814 | consumed samples: 314368 | consumed tokens: 643825664 | elapsed time per iteration (s): 5.37 | learning rate: 9.878E-04 | global batch size: 512 | lm loss: 4.923239E+00 | loss scale: 8192.0 | grad norm: 0.188 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.383 | TFLOPs: 72.04 | +[default7]: iteration 615/ 3814 | consumed samples: 314880 | consumed tokens: 644874240 | elapsed time per iteration (s): 5.38 | learning rate: 9.877E-04 | global batch size: 512 | lm loss: 4.923126E+00 | loss scale: 8192.0 | grad norm: 0.215 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.179 | TFLOPs: 71.89 | +[default7]: iteration 616/ 3814 | consumed samples: 315392 | consumed tokens: 645922816 | elapsed time per iteration (s): 5.38 | learning rate: 9.877E-04 | global batch size: 512 | lm loss: 4.943547E+00 | loss scale: 8192.0 | grad norm: 0.190 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.230 | TFLOPs: 71.93 | +[default7]: iteration 617/ 3814 | consumed samples: 315904 | consumed tokens: 646971392 | elapsed time per iteration (s): 5.36 | learning rate: 9.876E-04 | global batch size: 512 | lm loss: 4.908764E+00 | loss scale: 8192.0 | grad norm: 0.184 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.529 | TFLOPs: 72.15 | +[default7]: iteration 618/ 3814 | consumed samples: 316416 | consumed tokens: 648019968 | elapsed time per iteration (s): 5.38 | learning rate: 9.875E-04 | global batch size: 512 | lm loss: 4.919454E+00 | loss scale: 8192.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.132 | TFLOPs: 71.85 | +[default7]: iteration 619/ 3814 | consumed samples: 316928 | consumed tokens: 649068544 | elapsed time per iteration (s): 5.38 | learning rate: 9.874E-04 | global batch size: 512 | lm loss: 4.894423E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.146 | TFLOPs: 71.86 | +[default7]: iteration 620/ 3814 | consumed samples: 317440 | consumed tokens: 650117120 | elapsed time per iteration (s): 5.37 | learning rate: 9.873E-04 | global batch size: 512 | lm loss: 4.892057E+00 | loss scale: 8192.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.349 | TFLOPs: 72.02 | +[default7]: iteration 621/ 3814 | consumed samples: 317952 | consumed tokens: 651165696 | elapsed time per iteration (s): 5.36 | learning rate: 9.872E-04 | global batch size: 512 | lm loss: 4.880251E+00 | loss scale: 8192.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.441 | TFLOPs: 72.09 | +[default7]: iteration 622/ 3814 | consumed samples: 318464 | consumed tokens: 652214272 | elapsed time per iteration (s): 5.39 | learning rate: 9.871E-04 | global batch size: 512 | lm loss: 4.906385E+00 | loss scale: 8192.0 | grad norm: 0.177 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.948 | TFLOPs: 71.71 | +[default7]: iteration 623/ 3814 | consumed samples: 318976 | consumed tokens: 653262848 | elapsed time per iteration (s): 5.42 | learning rate: 9.870E-04 | global batch size: 512 | lm loss: 4.878119E+00 | loss scale: 8192.0 | grad norm: 0.163 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 624/ 3814 | consumed samples: 319488 | consumed tokens: 654311424 | elapsed time per iteration (s): 5.37 | learning rate: 9.869E-04 | global batch size: 512 | lm loss: 4.851025E+00 | loss scale: 8192.0 | grad norm: 0.158 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.303 | TFLOPs: 71.98 | +[default7]: iteration 625/ 3814 | consumed samples: 320000 | consumed tokens: 655360000 | elapsed time per iteration (s): 5.36 | learning rate: 9.868E-04 | global batch size: 512 | lm loss: 4.899256E+00 | loss scale: 8192.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.442 | TFLOPs: 72.09 | +[default7]: iteration 626/ 3814 | consumed samples: 320512 | consumed tokens: 656408576 | elapsed time per iteration (s): 5.37 | learning rate: 9.867E-04 | global batch size: 512 | lm loss: 4.862771E+00 | loss scale: 8192.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.295 | TFLOPs: 71.98 | +[default7]: iteration 627/ 3814 | consumed samples: 321024 | consumed tokens: 657457152 | elapsed time per iteration (s): 5.38 | learning rate: 9.866E-04 | global batch size: 512 | lm loss: 4.847731E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.254 | TFLOPs: 71.95 | +[default7]: iteration 628/ 3814 | consumed samples: 321536 | consumed tokens: 658505728 | elapsed time per iteration (s): 5.38 | learning rate: 9.865E-04 | global batch size: 512 | lm loss: 4.822765E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.133 | TFLOPs: 71.85 | +[default7]: iteration 629/ 3814 | consumed samples: 322048 | consumed tokens: 659554304 | elapsed time per iteration (s): 5.37 | learning rate: 9.864E-04 | global batch size: 512 | lm loss: 4.840973E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.406 | TFLOPs: 72.06 | +[default7]: iteration 630/ 3814 | consumed samples: 322560 | consumed tokens: 660602880 | elapsed time per iteration (s): 5.38 | learning rate: 9.863E-04 | global batch size: 512 | lm loss: 4.820263E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.116 | TFLOPs: 71.84 | +[default7]: iteration 631/ 3814 | consumed samples: 323072 | consumed tokens: 661651456 | elapsed time per iteration (s): 5.38 | learning rate: 9.862E-04 | global batch size: 512 | lm loss: 4.834890E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.219 | TFLOPs: 71.92 | +[default7]: iteration 632/ 3814 | consumed samples: 323584 | consumed tokens: 662700032 | elapsed time per iteration (s): 5.42 | learning rate: 9.861E-04 | global batch size: 512 | lm loss: 4.817055E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.465 | TFLOPs: 71.35 | +[default7]: iteration 633/ 3814 | consumed samples: 324096 | consumed tokens: 663748608 | elapsed time per iteration (s): 5.40 | learning rate: 9.860E-04 | global batch size: 512 | lm loss: 4.846851E+00 | loss scale: 8192.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.735 | TFLOPs: 71.55 | +[default7]: iteration 634/ 3814 | consumed samples: 324608 | consumed tokens: 664797184 | elapsed time per iteration (s): 5.39 | learning rate: 9.859E-04 | global batch size: 512 | lm loss: 4.832444E+00 | loss scale: 8192.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.937 | TFLOPs: 71.71 | +[default7]: iteration 635/ 3814 | consumed samples: 325120 | consumed tokens: 665845760 | elapsed time per iteration (s): 5.38 | learning rate: 9.858E-04 | global batch size: 512 | lm loss: 4.820893E+00 | loss scale: 8192.0 | grad norm: 0.181 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.111 | TFLOPs: 71.84 | +[default7]: iteration 636/ 3814 | consumed samples: 325632 | consumed tokens: 666894336 | elapsed time per iteration (s): 5.40 | learning rate: 9.857E-04 | global batch size: 512 | lm loss: 4.860029E+00 | loss scale: 8192.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.808 | TFLOPs: 71.61 | +[default7]: iteration 637/ 3814 | consumed samples: 326144 | consumed tokens: 667942912 | elapsed time per iteration (s): 5.39 | learning rate: 9.856E-04 | global batch size: 512 | lm loss: 4.790546E+00 | loss scale: 8192.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.047 | TFLOPs: 71.79 | +[default7]: iteration 638/ 3814 | consumed samples: 326656 | consumed tokens: 668991488 | elapsed time per iteration (s): 5.41 | learning rate: 9.855E-04 | global batch size: 512 | lm loss: 4.794250E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.679 | TFLOPs: 71.51 | +[default7]: iteration 639/ 3814 | consumed samples: 327168 | consumed tokens: 670040064 | elapsed time per iteration (s): 5.41 | learning rate: 9.854E-04 | global batch size: 512 | lm loss: 4.785256E+00 | loss scale: 8192.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.602 | TFLOPs: 71.45 | +[default7]: iteration 640/ 3814 | consumed samples: 327680 | consumed tokens: 671088640 | elapsed time per iteration (s): 5.43 | learning rate: 9.853E-04 | global batch size: 512 | lm loss: 4.763105E+00 | loss scale: 8192.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.290 | TFLOPs: 71.22 | +[default7]: iteration 641/ 3814 | consumed samples: 328192 | consumed tokens: 672137216 | elapsed time per iteration (s): 5.41 | learning rate: 9.852E-04 | global batch size: 512 | lm loss: 4.792428E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.677 | TFLOPs: 71.51 | +[default7]: iteration 642/ 3814 | consumed samples: 328704 | consumed tokens: 673185792 | elapsed time per iteration (s): 5.43 | learning rate: 9.851E-04 | global batch size: 512 | lm loss: 4.765979E+00 | loss scale: 8192.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.289 | TFLOPs: 71.22 | +[default7]: iteration 643/ 3814 | consumed samples: 329216 | consumed tokens: 674234368 | elapsed time per iteration (s): 5.41 | learning rate: 9.850E-04 | global batch size: 512 | lm loss: 4.788302E+00 | loss scale: 8192.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default7]: iteration 644/ 3814 | consumed samples: 329728 | consumed tokens: 675282944 | elapsed time per iteration (s): 5.39 | learning rate: 9.848E-04 | global batch size: 512 | lm loss: 4.762259E+00 | loss scale: 8192.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.074 | TFLOPs: 71.81 | +[default7]: iteration 645/ 3814 | consumed samples: 330240 | consumed tokens: 676331520 | elapsed time per iteration (s): 5.42 | learning rate: 9.847E-04 | global batch size: 512 | lm loss: 4.744947E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 646/ 3814 | consumed samples: 330752 | consumed tokens: 677380096 | elapsed time per iteration (s): 5.43 | learning rate: 9.846E-04 | global batch size: 512 | lm loss: 4.754422E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.309 | TFLOPs: 71.23 | +[default7]: iteration 647/ 3814 | consumed samples: 331264 | consumed tokens: 678428672 | elapsed time per iteration (s): 5.42 | learning rate: 9.845E-04 | global batch size: 512 | lm loss: 4.730485E+00 | loss scale: 8192.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.411 | TFLOPs: 71.31 | +[default7]: iteration 648/ 3814 | consumed samples: 331776 | consumed tokens: 679477248 | elapsed time per iteration (s): 5.41 | learning rate: 9.844E-04 | global batch size: 512 | lm loss: 4.752271E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 649/ 3814 | consumed samples: 332288 | consumed tokens: 680525824 | elapsed time per iteration (s): 5.39 | learning rate: 9.843E-04 | global batch size: 512 | lm loss: 4.740076E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.993 | TFLOPs: 71.75 | +[default7]: iteration 650/ 3814 | consumed samples: 332800 | consumed tokens: 681574400 | elapsed time per iteration (s): 5.38 | learning rate: 9.842E-04 | global batch size: 512 | lm loss: 4.761894E+00 | loss scale: 8192.0 | grad norm: 0.151 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.083 | TFLOPs: 71.82 | +[default7]: iteration 651/ 3814 | consumed samples: 333312 | consumed tokens: 682622976 | elapsed time per iteration (s): 5.42 | learning rate: 9.841E-04 | global batch size: 512 | lm loss: 4.720678E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.516 | TFLOPs: 71.39 | +[default7]: iteration 652/ 3814 | consumed samples: 333824 | consumed tokens: 683671552 | elapsed time per iteration (s): 5.42 | learning rate: 9.840E-04 | global batch size: 512 | lm loss: 4.785983E+00 | loss scale: 8192.0 | grad norm: 0.152 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.499 | TFLOPs: 71.38 | +[default7]: iteration 653/ 3814 | consumed samples: 334336 | consumed tokens: 684720128 | elapsed time per iteration (s): 5.41 | learning rate: 9.839E-04 | global batch size: 512 | lm loss: 4.763503E+00 | loss scale: 8192.0 | grad norm: 0.147 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 654/ 3814 | consumed samples: 334848 | consumed tokens: 685768704 | elapsed time per iteration (s): 5.39 | learning rate: 9.838E-04 | global batch size: 512 | lm loss: 4.743333E+00 | loss scale: 8192.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.960 | TFLOPs: 71.72 | +[default7]: iteration 655/ 3814 | consumed samples: 335360 | consumed tokens: 686817280 | elapsed time per iteration (s): 5.41 | learning rate: 9.837E-04 | global batch size: 512 | lm loss: 4.731891E+00 | loss scale: 8192.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.720 | TFLOPs: 71.54 | +[default7]: iteration 656/ 3814 | consumed samples: 335872 | consumed tokens: 687865856 | elapsed time per iteration (s): 5.41 | learning rate: 9.836E-04 | global batch size: 512 | lm loss: 4.697668E+00 | loss scale: 8192.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.635 | TFLOPs: 71.48 | +[default7]: iteration 657/ 3814 | consumed samples: 336384 | consumed tokens: 688914432 | elapsed time per iteration (s): 5.40 | learning rate: 9.834E-04 | global batch size: 512 | lm loss: 4.744778E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.807 | TFLOPs: 71.61 | +[default7]: iteration 658/ 3814 | consumed samples: 336896 | consumed tokens: 689963008 | elapsed time per iteration (s): 5.40 | learning rate: 9.833E-04 | global batch size: 512 | lm loss: 4.729640E+00 | loss scale: 8192.0 | grad norm: 0.143 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.896 | TFLOPs: 71.68 | +[default7]: iteration 659/ 3814 | consumed samples: 337408 | consumed tokens: 691011584 | elapsed time per iteration (s): 5.42 | learning rate: 9.832E-04 | global batch size: 512 | lm loss: 4.680609E+00 | loss scale: 8192.0 | grad norm: 0.147 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.548 | TFLOPs: 71.41 | +[default7]: iteration 660/ 3814 | consumed samples: 337920 | consumed tokens: 692060160 | elapsed time per iteration (s): 5.43 | learning rate: 9.831E-04 | global batch size: 512 | lm loss: 4.691784E+00 | loss scale: 8192.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.318 | TFLOPs: 71.24 | +[default7]: iteration 661/ 3814 | consumed samples: 338432 | consumed tokens: 693108736 | elapsed time per iteration (s): 5.44 | learning rate: 9.830E-04 | global batch size: 512 | lm loss: 4.709825E+00 | loss scale: 8192.0 | grad norm: 0.159 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.087 | TFLOPs: 71.06 | +[default7]: iteration 662/ 3814 | consumed samples: 338944 | consumed tokens: 694157312 | elapsed time per iteration (s): 5.41 | learning rate: 9.829E-04 | global batch size: 512 | lm loss: 4.720758E+00 | loss scale: 8192.0 | grad norm: 0.172 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.638 | TFLOPs: 71.48 | +[default7]: iteration 663/ 3814 | consumed samples: 339456 | consumed tokens: 695205888 | elapsed time per iteration (s): 5.40 | learning rate: 9.828E-04 | global batch size: 512 | lm loss: 4.717002E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.844 | TFLOPs: 71.64 | +[default7]: iteration 664/ 3814 | consumed samples: 339968 | consumed tokens: 696254464 | elapsed time per iteration (s): 5.40 | learning rate: 9.827E-04 | global batch size: 512 | lm loss: 4.703676E+00 | loss scale: 8192.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.853 | TFLOPs: 71.64 | +[default7]: iteration 665/ 3814 | consumed samples: 340480 | consumed tokens: 697303040 | elapsed time per iteration (s): 5.43 | learning rate: 9.826E-04 | global batch size: 512 | lm loss: 4.704212E+00 | loss scale: 8192.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.321 | TFLOPs: 71.24 | +[default7]: iteration 666/ 3814 | consumed samples: 340992 | consumed tokens: 698351616 | elapsed time per iteration (s): 5.44 | learning rate: 9.824E-04 | global batch size: 512 | lm loss: 4.677293E+00 | loss scale: 8192.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.068 | TFLOPs: 71.05 | +[default7]: iteration 667/ 3814 | consumed samples: 341504 | consumed tokens: 699400192 | elapsed time per iteration (s): 5.40 | learning rate: 9.823E-04 | global batch size: 512 | lm loss: 4.664507E+00 | loss scale: 8192.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.861 | TFLOPs: 71.65 | +[default7]: iteration 668/ 3814 | consumed samples: 342016 | consumed tokens: 700448768 | elapsed time per iteration (s): 5.39 | learning rate: 9.822E-04 | global batch size: 512 | lm loss: 4.654910E+00 | loss scale: 8192.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.973 | TFLOPs: 71.73 | +[default7]: iteration 669/ 3814 | consumed samples: 342528 | consumed tokens: 701497344 | elapsed time per iteration (s): 5.40 | learning rate: 9.821E-04 | global batch size: 512 | lm loss: 4.678543E+00 | loss scale: 8192.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.882 | TFLOPs: 71.67 | +[default7]: iteration 670/ 3814 | consumed samples: 343040 | consumed tokens: 702545920 | elapsed time per iteration (s): 5.40 | learning rate: 9.820E-04 | global batch size: 512 | lm loss: 4.658674E+00 | loss scale: 8192.0 | grad norm: 0.148 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.844 | TFLOPs: 71.64 | +[default7]: iteration 671/ 3814 | consumed samples: 343552 | consumed tokens: 703594496 | elapsed time per iteration (s): 5.42 | learning rate: 9.819E-04 | global batch size: 512 | lm loss: 4.675499E+00 | loss scale: 8192.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.409 | TFLOPs: 71.31 | +[default7]: iteration 672/ 3814 | consumed samples: 344064 | consumed tokens: 704643072 | elapsed time per iteration (s): 5.38 | learning rate: 9.818E-04 | global batch size: 512 | lm loss: 4.627010E+00 | loss scale: 8192.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.179 | TFLOPs: 71.89 | +[default7]: iteration 673/ 3814 | consumed samples: 344576 | consumed tokens: 705691648 | elapsed time per iteration (s): 5.38 | learning rate: 9.816E-04 | global batch size: 512 | lm loss: 4.641589E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.106 | TFLOPs: 71.83 | +[default7]: iteration 674/ 3814 | consumed samples: 345088 | consumed tokens: 706740224 | elapsed time per iteration (s): 5.39 | learning rate: 9.815E-04 | global batch size: 512 | lm loss: 4.622537E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.955 | TFLOPs: 71.72 | +[default7]: iteration 675/ 3814 | consumed samples: 345600 | consumed tokens: 707788800 | elapsed time per iteration (s): 5.41 | learning rate: 9.814E-04 | global batch size: 512 | lm loss: 4.632170E+00 | loss scale: 8192.0 | grad norm: 0.144 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.637 | TFLOPs: 71.48 | +[default7]: iteration 676/ 3814 | consumed samples: 346112 | consumed tokens: 708837376 | elapsed time per iteration (s): 5.40 | learning rate: 9.813E-04 | global batch size: 512 | lm loss: 4.642589E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.754 | TFLOPs: 71.57 | +[default7]: iteration 677/ 3814 | consumed samples: 346624 | consumed tokens: 709885952 | elapsed time per iteration (s): 5.39 | learning rate: 9.812E-04 | global batch size: 512 | lm loss: 4.611246E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.039 | TFLOPs: 71.78 | +[default7]: iteration 678/ 3814 | consumed samples: 347136 | consumed tokens: 710934528 | elapsed time per iteration (s): 5.40 | learning rate: 9.811E-04 | global batch size: 512 | lm loss: 4.612427E+00 | loss scale: 8192.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.857 | TFLOPs: 71.65 | +[default7]: iteration 679/ 3814 | consumed samples: 347648 | consumed tokens: 711983104 | elapsed time per iteration (s): 5.40 | learning rate: 9.809E-04 | global batch size: 512 | lm loss: 4.601832E+00 | loss scale: 8192.0 | grad norm: 0.149 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 680/ 3814 | consumed samples: 348160 | consumed tokens: 713031680 | elapsed time per iteration (s): 5.39 | learning rate: 9.808E-04 | global batch size: 512 | lm loss: 4.618409E+00 | loss scale: 8192.0 | grad norm: 0.144 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.021 | TFLOPs: 71.77 | +[default7]: iteration 681/ 3814 | consumed samples: 348672 | consumed tokens: 714080256 | elapsed time per iteration (s): 5.39 | learning rate: 9.807E-04 | global batch size: 512 | lm loss: 4.617010E+00 | loss scale: 8192.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.010 | TFLOPs: 71.76 | +[default7]: iteration 682/ 3814 | consumed samples: 349184 | consumed tokens: 715128832 | elapsed time per iteration (s): 5.40 | learning rate: 9.806E-04 | global batch size: 512 | lm loss: 4.625637E+00 | loss scale: 8192.0 | grad norm: 0.163 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.842 | TFLOPs: 71.64 | +[default7]: iteration 683/ 3814 | consumed samples: 349696 | consumed tokens: 716177408 | elapsed time per iteration (s): 5.40 | learning rate: 9.805E-04 | global batch size: 512 | lm loss: 4.620791E+00 | loss scale: 8192.0 | grad norm: 0.181 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.805 | TFLOPs: 71.61 | +[default7]: iteration 684/ 3814 | consumed samples: 350208 | consumed tokens: 717225984 | elapsed time per iteration (s): 5.41 | learning rate: 9.804E-04 | global batch size: 512 | lm loss: 4.592574E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 685/ 3814 | consumed samples: 350720 | consumed tokens: 718274560 | elapsed time per iteration (s): 5.38 | learning rate: 9.802E-04 | global batch size: 512 | lm loss: 4.599831E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.160 | TFLOPs: 71.88 | +[default7]: iteration 686/ 3814 | consumed samples: 351232 | consumed tokens: 719323136 | elapsed time per iteration (s): 5.39 | learning rate: 9.801E-04 | global batch size: 512 | lm loss: 4.562319E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.066 | TFLOPs: 71.80 | +[default7]: iteration 687/ 3814 | consumed samples: 351744 | consumed tokens: 720371712 | elapsed time per iteration (s): 5.38 | learning rate: 9.800E-04 | global batch size: 512 | lm loss: 4.575446E+00 | loss scale: 8192.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.217 | TFLOPs: 71.92 | +[default7]: iteration 688/ 3814 | consumed samples: 352256 | consumed tokens: 721420288 | elapsed time per iteration (s): 5.39 | learning rate: 9.799E-04 | global batch size: 512 | lm loss: 4.566552E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.027 | TFLOPs: 71.77 | +[default7]: iteration 689/ 3814 | consumed samples: 352768 | consumed tokens: 722468864 | elapsed time per iteration (s): 5.39 | learning rate: 9.798E-04 | global batch size: 512 | lm loss: 4.563294E+00 | loss scale: 8192.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.988 | TFLOPs: 71.75 | +[default7]: iteration 690/ 3814 | consumed samples: 353280 | consumed tokens: 723517440 | elapsed time per iteration (s): 5.39 | learning rate: 9.796E-04 | global batch size: 512 | lm loss: 4.548184E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.999 | TFLOPs: 71.75 | +[default7]: iteration 691/ 3814 | consumed samples: 353792 | consumed tokens: 724566016 | elapsed time per iteration (s): 5.40 | learning rate: 9.795E-04 | global batch size: 512 | lm loss: 4.529513E+00 | loss scale: 8192.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.897 | TFLOPs: 71.68 | +[default7]: iteration 692/ 3814 | consumed samples: 354304 | consumed tokens: 725614592 | elapsed time per iteration (s): 5.38 | learning rate: 9.794E-04 | global batch size: 512 | lm loss: 4.535956E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.213 | TFLOPs: 71.92 | +[default7]: iteration 693/ 3814 | consumed samples: 354816 | consumed tokens: 726663168 | elapsed time per iteration (s): 5.39 | learning rate: 9.793E-04 | global batch size: 512 | lm loss: 4.527215E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.058 | TFLOPs: 71.80 | +[default7]: iteration 694/ 3814 | consumed samples: 355328 | consumed tokens: 727711744 | elapsed time per iteration (s): 5.39 | learning rate: 9.791E-04 | global batch size: 512 | lm loss: 4.513956E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.915 | TFLOPs: 71.69 | +[default7]: iteration 695/ 3814 | consumed samples: 355840 | consumed tokens: 728760320 | elapsed time per iteration (s): 5.40 | learning rate: 9.790E-04 | global batch size: 512 | lm loss: 4.527828E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.834 | TFLOPs: 71.63 | +[default7]: iteration 696/ 3814 | consumed samples: 356352 | consumed tokens: 729808896 | elapsed time per iteration (s): 5.39 | learning rate: 9.789E-04 | global batch size: 512 | lm loss: 4.486448E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.981 | TFLOPs: 71.74 | +[default7]: iteration 697/ 3814 | consumed samples: 356864 | consumed tokens: 730857472 | elapsed time per iteration (s): 5.42 | learning rate: 9.788E-04 | global batch size: 512 | lm loss: 4.535080E+00 | loss scale: 8192.0 | grad norm: 0.174 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 698/ 3814 | consumed samples: 357376 | consumed tokens: 731906048 | elapsed time per iteration (s): 5.40 | learning rate: 9.786E-04 | global batch size: 512 | lm loss: 4.548572E+00 | loss scale: 8192.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.744 | TFLOPs: 71.56 | +[default7]: iteration 699/ 3814 | consumed samples: 357888 | consumed tokens: 732954624 | elapsed time per iteration (s): 5.41 | learning rate: 9.785E-04 | global batch size: 512 | lm loss: 4.518558E+00 | loss scale: 8192.0 | grad norm: 0.176 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.580 | TFLOPs: 71.44 | +[default7]: iteration 700/ 3814 | consumed samples: 358400 | consumed tokens: 734003200 | elapsed time per iteration (s): 5.40 | learning rate: 9.784E-04 | global batch size: 512 | lm loss: 4.547487E+00 | loss scale: 8192.0 | grad norm: 0.153 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.800 | TFLOPs: 71.60 | +[default7]: iteration 701/ 3814 | consumed samples: 358912 | consumed tokens: 735051776 | elapsed time per iteration (s): 5.41 | learning rate: 9.783E-04 | global batch size: 512 | lm loss: 4.553901E+00 | loss scale: 8192.0 | grad norm: 0.225 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 702/ 3814 | consumed samples: 359424 | consumed tokens: 736100352 | elapsed time per iteration (s): 5.42 | learning rate: 9.781E-04 | global batch size: 512 | lm loss: 4.586411E+00 | loss scale: 8192.0 | grad norm: 0.187 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 703/ 3814 | consumed samples: 359936 | consumed tokens: 737148928 | elapsed time per iteration (s): 5.40 | learning rate: 9.780E-04 | global batch size: 512 | lm loss: 4.570933E+00 | loss scale: 8192.0 | grad norm: 0.214 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 704/ 3814 | consumed samples: 360448 | consumed tokens: 738197504 | elapsed time per iteration (s): 5.39 | learning rate: 9.779E-04 | global batch size: 512 | lm loss: 4.593032E+00 | loss scale: 8192.0 | grad norm: 0.226 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.042 | TFLOPs: 71.79 | +[default7]: iteration 705/ 3814 | consumed samples: 360960 | consumed tokens: 739246080 | elapsed time per iteration (s): 5.40 | learning rate: 9.778E-04 | global batch size: 512 | lm loss: 4.616982E+00 | loss scale: 8192.0 | grad norm: 0.255 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.901 | TFLOPs: 71.68 | +[default7]: iteration 706/ 3814 | consumed samples: 361472 | consumed tokens: 740294656 | elapsed time per iteration (s): 5.39 | learning rate: 9.776E-04 | global batch size: 512 | lm loss: 4.604273E+00 | loss scale: 8192.0 | grad norm: 0.178 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.001 | TFLOPs: 71.76 | +[default7]: iteration 707/ 3814 | consumed samples: 361984 | consumed tokens: 741343232 | elapsed time per iteration (s): 5.41 | learning rate: 9.775E-04 | global batch size: 512 | lm loss: 4.561805E+00 | loss scale: 8192.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.629 | TFLOPs: 71.47 | +[default7]: iteration 708/ 3814 | consumed samples: 362496 | consumed tokens: 742391808 | elapsed time per iteration (s): 5.41 | learning rate: 9.774E-04 | global batch size: 512 | lm loss: 4.543706E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 709/ 3814 | consumed samples: 363008 | consumed tokens: 743440384 | elapsed time per iteration (s): 5.39 | learning rate: 9.773E-04 | global batch size: 512 | lm loss: 4.532090E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.976 | TFLOPs: 71.74 | +[default7]: iteration 710/ 3814 | consumed samples: 363520 | consumed tokens: 744488960 | elapsed time per iteration (s): 5.40 | learning rate: 9.771E-04 | global batch size: 512 | lm loss: 4.516899E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.842 | TFLOPs: 71.64 | +[default7]: iteration 711/ 3814 | consumed samples: 364032 | consumed tokens: 745537536 | elapsed time per iteration (s): 5.40 | learning rate: 9.770E-04 | global batch size: 512 | lm loss: 4.515840E+00 | loss scale: 8192.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.880 | TFLOPs: 71.66 | +[default7]: iteration 712/ 3814 | consumed samples: 364544 | consumed tokens: 746586112 | elapsed time per iteration (s): 5.40 | learning rate: 9.769E-04 | global batch size: 512 | lm loss: 4.482007E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.870 | TFLOPs: 71.66 | +[default7]: iteration 713/ 3814 | consumed samples: 365056 | consumed tokens: 747634688 | elapsed time per iteration (s): 5.43 | learning rate: 9.767E-04 | global batch size: 512 | lm loss: 4.481768E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.326 | TFLOPs: 71.25 | +[default7]: iteration 714/ 3814 | consumed samples: 365568 | consumed tokens: 748683264 | elapsed time per iteration (s): 5.40 | learning rate: 9.766E-04 | global batch size: 512 | lm loss: 4.500340E+00 | loss scale: 8192.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.870 | TFLOPs: 71.66 | +[default7]: iteration 715/ 3814 | consumed samples: 366080 | consumed tokens: 749731840 | elapsed time per iteration (s): 5.40 | learning rate: 9.765E-04 | global batch size: 512 | lm loss: 4.442912E+00 | loss scale: 8192.0 | grad norm: 0.065 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.872 | TFLOPs: 71.66 | +[default7]: iteration 716/ 3814 | consumed samples: 366592 | consumed tokens: 750780416 | elapsed time per iteration (s): 5.40 | learning rate: 9.763E-04 | global batch size: 512 | lm loss: 4.476936E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.831 | TFLOPs: 71.63 | +[default7]: iteration 717/ 3814 | consumed samples: 367104 | consumed tokens: 751828992 | elapsed time per iteration (s): 5.40 | learning rate: 9.762E-04 | global batch size: 512 | lm loss: 4.449780E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.794 | TFLOPs: 71.60 | +[default7]: iteration 718/ 3814 | consumed samples: 367616 | consumed tokens: 752877568 | elapsed time per iteration (s): 5.39 | learning rate: 9.761E-04 | global batch size: 512 | lm loss: 4.493495E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.037 | TFLOPs: 71.78 | +[default7]: iteration 719/ 3814 | consumed samples: 368128 | consumed tokens: 753926144 | elapsed time per iteration (s): 5.41 | learning rate: 9.760E-04 | global batch size: 512 | lm loss: 4.465981E+00 | loss scale: 8192.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.628 | TFLOPs: 71.47 | +[default7]: iteration 720/ 3814 | consumed samples: 368640 | consumed tokens: 754974720 | elapsed time per iteration (s): 5.40 | learning rate: 9.758E-04 | global batch size: 512 | lm loss: 4.467344E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.876 | TFLOPs: 71.66 | +[default7]: iteration 721/ 3814 | consumed samples: 369152 | consumed tokens: 756023296 | elapsed time per iteration (s): 5.40 | learning rate: 9.757E-04 | global batch size: 512 | lm loss: 4.460108E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.821 | TFLOPs: 71.62 | +[default7]: iteration 722/ 3814 | consumed samples: 369664 | consumed tokens: 757071872 | elapsed time per iteration (s): 5.41 | learning rate: 9.756E-04 | global batch size: 512 | lm loss: 4.440147E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 723/ 3814 | consumed samples: 370176 | consumed tokens: 758120448 | elapsed time per iteration (s): 5.41 | learning rate: 9.754E-04 | global batch size: 512 | lm loss: 4.452761E+00 | loss scale: 8192.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.689 | TFLOPs: 71.52 | +[default7]: iteration 724/ 3814 | consumed samples: 370688 | consumed tokens: 759169024 | elapsed time per iteration (s): 5.40 | learning rate: 9.753E-04 | global batch size: 512 | lm loss: 4.434864E+00 | loss scale: 8192.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.858 | TFLOPs: 71.65 | +[default7]: iteration 725/ 3814 | consumed samples: 371200 | consumed tokens: 760217600 | elapsed time per iteration (s): 5.41 | learning rate: 9.752E-04 | global batch size: 512 | lm loss: 4.441259E+00 | loss scale: 8192.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.632 | TFLOPs: 71.48 | +[default7]: iteration 726/ 3814 | consumed samples: 371712 | consumed tokens: 761266176 | elapsed time per iteration (s): 5.41 | learning rate: 9.750E-04 | global batch size: 512 | lm loss: 4.466861E+00 | loss scale: 8192.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.567 | TFLOPs: 71.43 | +[default7]: iteration 727/ 3814 | consumed samples: 372224 | consumed tokens: 762314752 | elapsed time per iteration (s): 5.38 | learning rate: 9.749E-04 | global batch size: 512 | lm loss: 4.423875E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.208 | TFLOPs: 71.91 | +[default7]: iteration 728/ 3814 | consumed samples: 372736 | consumed tokens: 763363328 | elapsed time per iteration (s): 5.38 | learning rate: 9.748E-04 | global batch size: 512 | lm loss: 4.433309E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.106 | TFLOPs: 71.83 | +[default7]: iteration 729/ 3814 | consumed samples: 373248 | consumed tokens: 764411904 | elapsed time per iteration (s): 5.40 | learning rate: 9.746E-04 | global batch size: 512 | lm loss: 4.426488E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.813 | TFLOPs: 71.61 | +[default7]: iteration 730/ 3814 | consumed samples: 373760 | consumed tokens: 765460480 | elapsed time per iteration (s): 5.41 | learning rate: 9.745E-04 | global batch size: 512 | lm loss: 4.416665E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 731/ 3814 | consumed samples: 374272 | consumed tokens: 766509056 | elapsed time per iteration (s): 5.39 | learning rate: 9.743E-04 | global batch size: 512 | lm loss: 4.420526E+00 | loss scale: 8192.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.065 | TFLOPs: 71.80 | +[default7]: iteration 732/ 3814 | consumed samples: 374784 | consumed tokens: 767557632 | elapsed time per iteration (s): 5.40 | learning rate: 9.742E-04 | global batch size: 512 | lm loss: 4.431278E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.788 | TFLOPs: 71.59 | +[default7]: iteration 733/ 3814 | consumed samples: 375296 | consumed tokens: 768606208 | elapsed time per iteration (s): 5.39 | learning rate: 9.741E-04 | global batch size: 512 | lm loss: 4.403287E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.004 | TFLOPs: 71.76 | +[default7]: iteration 734/ 3814 | consumed samples: 375808 | consumed tokens: 769654784 | elapsed time per iteration (s): 5.40 | learning rate: 9.739E-04 | global batch size: 512 | lm loss: 4.415788E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.759 | TFLOPs: 71.57 | +[default7]: iteration 735/ 3814 | consumed samples: 376320 | consumed tokens: 770703360 | elapsed time per iteration (s): 5.43 | learning rate: 9.738E-04 | global batch size: 512 | lm loss: 4.375799E+00 | loss scale: 8192.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.337 | TFLOPs: 71.25 | +[default7]: iteration 736/ 3814 | consumed samples: 376832 | consumed tokens: 771751936 | elapsed time per iteration (s): 5.40 | learning rate: 9.737E-04 | global batch size: 512 | lm loss: 4.394394E+00 | loss scale: 8192.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.771 | TFLOPs: 71.58 | +[default7]: iteration 737/ 3814 | consumed samples: 377344 | consumed tokens: 772800512 | elapsed time per iteration (s): 5.38 | learning rate: 9.735E-04 | global batch size: 512 | lm loss: 4.374678E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.091 | TFLOPs: 71.82 | +[default7]: iteration 738/ 3814 | consumed samples: 377856 | consumed tokens: 773849088 | elapsed time per iteration (s): 5.38 | learning rate: 9.734E-04 | global batch size: 512 | lm loss: 4.393581E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.138 | TFLOPs: 71.86 | +[default7]: iteration 739/ 3814 | consumed samples: 378368 | consumed tokens: 774897664 | elapsed time per iteration (s): 5.39 | learning rate: 9.732E-04 | global batch size: 512 | lm loss: 4.373276E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.061 | TFLOPs: 71.80 | +[default7]: iteration 740/ 3814 | consumed samples: 378880 | consumed tokens: 775946240 | elapsed time per iteration (s): 5.41 | learning rate: 9.731E-04 | global batch size: 512 | lm loss: 4.389532E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.685 | TFLOPs: 71.52 | +[default7]: iteration 741/ 3814 | consumed samples: 379392 | consumed tokens: 776994816 | elapsed time per iteration (s): 5.39 | learning rate: 9.730E-04 | global batch size: 512 | lm loss: 4.371696E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.946 | TFLOPs: 71.71 | +[default7]: iteration 742/ 3814 | consumed samples: 379904 | consumed tokens: 778043392 | elapsed time per iteration (s): 5.39 | learning rate: 9.728E-04 | global batch size: 512 | lm loss: 4.370083E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.037 | TFLOPs: 71.78 | +[default7]: iteration 743/ 3814 | consumed samples: 380416 | consumed tokens: 779091968 | elapsed time per iteration (s): 5.38 | learning rate: 9.727E-04 | global batch size: 512 | lm loss: 4.368326E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.118 | TFLOPs: 71.84 | +[default7]: iteration 744/ 3814 | consumed samples: 380928 | consumed tokens: 780140544 | elapsed time per iteration (s): 5.39 | learning rate: 9.725E-04 | global batch size: 512 | lm loss: 4.358749E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.980 | TFLOPs: 71.74 | +[default7]: iteration 745/ 3814 | consumed samples: 381440 | consumed tokens: 781189120 | elapsed time per iteration (s): 5.39 | learning rate: 9.724E-04 | global batch size: 512 | lm loss: 4.361899E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.981 | TFLOPs: 71.74 | +[default7]: iteration 746/ 3814 | consumed samples: 381952 | consumed tokens: 782237696 | elapsed time per iteration (s): 5.37 | learning rate: 9.723E-04 | global batch size: 512 | lm loss: 4.346409E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.427 | TFLOPs: 72.08 | +[default7]: iteration 747/ 3814 | consumed samples: 382464 | consumed tokens: 783286272 | elapsed time per iteration (s): 5.39 | learning rate: 9.721E-04 | global batch size: 512 | lm loss: 4.389390E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.036 | TFLOPs: 71.78 | +[default7]: iteration 748/ 3814 | consumed samples: 382976 | consumed tokens: 784334848 | elapsed time per iteration (s): 5.39 | learning rate: 9.720E-04 | global batch size: 512 | lm loss: 4.370604E+00 | loss scale: 8192.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.967 | TFLOPs: 71.73 | +[default7]: iteration 749/ 3814 | consumed samples: 383488 | consumed tokens: 785383424 | elapsed time per iteration (s): 5.40 | learning rate: 9.718E-04 | global batch size: 512 | lm loss: 4.353172E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.871 | TFLOPs: 71.66 | +[default7]: iteration 750/ 3814 | consumed samples: 384000 | consumed tokens: 786432000 | elapsed time per iteration (s): 5.41 | learning rate: 9.717E-04 | global batch size: 512 | lm loss: 4.353770E+00 | loss scale: 8192.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.689 | TFLOPs: 71.52 | +[default0]:saving checkpoint at iteration 750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 07:27:24,638] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step750 is begin to save! +[default0]:[2023-02-16 07:27:24,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:24,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_01-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:24,795] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:24,795] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:24,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:24,797] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:24,860] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:24,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:24,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:24,861] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_05-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:24,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:24,926] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_06-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:24,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:24,928] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:24,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:24,985] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:24,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:24,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,041] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,095] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,038] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,092] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,151] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,147] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,207] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,267] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,268] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,329] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,388] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,376] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,452] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,510] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,510] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,432] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,489] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,490] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,568] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,553] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,554] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,612] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,613] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,700] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,676] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_19-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,761] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,761] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_20-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,822] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_20-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,797] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_21-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,876] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,855] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_22-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:25,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,909] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:25,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,932] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_23-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:25,990] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:25,990] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:25,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:25,968] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,026] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,026] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,049] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,050] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,113] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,113] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:26,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,089] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,171] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_27-model_00-model_states.pt. +[default1]:[2023-02-16 07:27:26,148] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,148] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_27-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,205] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,225] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_28-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,282] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:26,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,262] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,342] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:26,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,374] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,459] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,522] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,523] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:26,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,498] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 07:27:26,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,583] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 07:27:26,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,562] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_35-model_01-model_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:27:26,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,688] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,689] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 07:27:26,689] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 07:27:26,691] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_00_model_states.pt +[default0]:[2023-02-16 07:27:26,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 07:27:26,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:27:26,674] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,674] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 07:27:26,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 07:27:26,677] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_01_model_states.pt +[default1]:[2023-02-16 07:27:26,677] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 07:27:26,679] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:27:26,722] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:27:27,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,163] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,163] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,225] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,225] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,219] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,219] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,219] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,238] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,238] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,238] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,238] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,238] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,278] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,278] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,258] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,258] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,261] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,261] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,261] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,241] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,241] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,231] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,231] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,272] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,272] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,272] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,247] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,247] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,247] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,270] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,270] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,287] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,287] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,287] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,279] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,279] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,279] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,278] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,278] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,318] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,319] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,319] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,276] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,276] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,294] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,294] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,332] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,332] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,294] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,294] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,294] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,355] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,355] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,355] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,303] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,303] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,315] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,315] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,369] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,369] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,290] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,290] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,290] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,342] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,342] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,317] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,317] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,305] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,305] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,305] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,318] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,318] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,318] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,326] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,326] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,326] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,333] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,333] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,308] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,308] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,308] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,347] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,347] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,347] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,339] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,339] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,379] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,380] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,380] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,406] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,406] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,406] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,375] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,375] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,360] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,360] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,395] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,396] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,335] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,335] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,395] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,395] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,395] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,413] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,413] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,413] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,341] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,341] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,365] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,365] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,365] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,328] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,328] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,328] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,346] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,346] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,343] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,343] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,372] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,372] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,424] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,424] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,374] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,374] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default3]:[2023-02-16 07:27:27,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:27:27,429] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:27:27,430] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,364] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,364] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,407] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,407] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,399] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,399] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,399] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default4]:[2023-02-16 07:27:27,475] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:27:27,476] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:27:27,476] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:[2023-02-16 07:27:27,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:27:27,425] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:27:27,425] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,427] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,427] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,513] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,513] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,513] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default6]:[2023-02-16 07:27:27,451] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:27:27,451] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:27:27,451] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default1]:[2023-02-16 07:27:27,446] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:27:27,447] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:27:27,447] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default0]:[2023-02-16 07:27:27,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:27:27,517] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:27:27,517] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default2]:[2023-02-16 07:27:27,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:27:27,486] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:27:27,486] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default5]:[2023-02-16 07:27:27,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:27:27,579] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step750/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:27:27,579] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step750 is ready now! +[default7]:time (ms) | save-checkpoint: 2942.37 +[default0]: successfully saved checkpoint at iteration 750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]: iteration 751/ 3814 | consumed samples: 384512 | consumed tokens: 787480576 | elapsed time per iteration (s): 8.33 | learning rate: 9.716E-04 | global batch size: 512 | lm loss: 4.356687E+00 | loss scale: 8192.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.479 | TFLOPs: 46.44 | +[default7]: iteration 752/ 3814 | consumed samples: 385024 | consumed tokens: 788529152 | elapsed time per iteration (s): 5.39 | learning rate: 9.714E-04 | global batch size: 512 | lm loss: 4.360678E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.025 | TFLOPs: 71.77 | +[default7]: iteration 753/ 3814 | consumed samples: 385536 | consumed tokens: 789577728 | elapsed time per iteration (s): 5.41 | learning rate: 9.713E-04 | global batch size: 512 | lm loss: 4.367179E+00 | loss scale: 8192.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.715 | TFLOPs: 71.54 | +[default7]: iteration 754/ 3814 | consumed samples: 386048 | consumed tokens: 790626304 | elapsed time per iteration (s): 5.40 | learning rate: 9.711E-04 | global batch size: 512 | lm loss: 4.342747E+00 | loss scale: 8192.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.880 | TFLOPs: 71.66 | +[default7]: iteration 755/ 3814 | consumed samples: 386560 | consumed tokens: 791674880 | elapsed time per iteration (s): 5.39 | learning rate: 9.710E-04 | global batch size: 512 | lm loss: 4.347926E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.028 | TFLOPs: 71.78 | +[default7]: iteration 756/ 3814 | consumed samples: 387072 | consumed tokens: 792723456 | elapsed time per iteration (s): 5.40 | learning rate: 9.708E-04 | global batch size: 512 | lm loss: 4.349040E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.791 | TFLOPs: 71.60 | +[default7]: iteration 757/ 3814 | consumed samples: 387584 | consumed tokens: 793772032 | elapsed time per iteration (s): 5.41 | learning rate: 9.707E-04 | global batch size: 512 | lm loss: 4.357334E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.644 | TFLOPs: 71.49 | +[default7]: iteration 758/ 3814 | consumed samples: 388096 | consumed tokens: 794820608 | elapsed time per iteration (s): 5.41 | learning rate: 9.705E-04 | global batch size: 512 | lm loss: 4.326098E+00 | loss scale: 8192.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.665 | TFLOPs: 71.50 | +[default7]: iteration 759/ 3814 | consumed samples: 388608 | consumed tokens: 795869184 | elapsed time per iteration (s): 5.40 | learning rate: 9.704E-04 | global batch size: 512 | lm loss: 4.306870E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.768 | TFLOPs: 71.58 | +[default7]: iteration 760/ 3814 | consumed samples: 389120 | consumed tokens: 796917760 | elapsed time per iteration (s): 5.40 | learning rate: 9.703E-04 | global batch size: 512 | lm loss: 4.293867E+00 | loss scale: 8192.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.885 | TFLOPs: 71.67 | +[default7]: iteration 761/ 3814 | consumed samples: 389632 | consumed tokens: 797966336 | elapsed time per iteration (s): 5.41 | learning rate: 9.701E-04 | global batch size: 512 | lm loss: 4.324306E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.637 | TFLOPs: 71.48 | +[default7]: iteration 762/ 3814 | consumed samples: 390144 | consumed tokens: 799014912 | elapsed time per iteration (s): 5.41 | learning rate: 9.700E-04 | global batch size: 512 | lm loss: 4.315898E+00 | loss scale: 8192.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 763/ 3814 | consumed samples: 390656 | consumed tokens: 800063488 | elapsed time per iteration (s): 5.41 | learning rate: 9.698E-04 | global batch size: 512 | lm loss: 4.308526E+00 | loss scale: 8192.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.608 | TFLOPs: 71.46 | +[default7]: iteration 764/ 3814 | consumed samples: 391168 | consumed tokens: 801112064 | elapsed time per iteration (s): 5.43 | learning rate: 9.697E-04 | global batch size: 512 | lm loss: 4.306492E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.212 | TFLOPs: 71.16 | +[default7]: iteration 765/ 3814 | consumed samples: 391680 | consumed tokens: 802160640 | elapsed time per iteration (s): 5.40 | learning rate: 9.695E-04 | global batch size: 512 | lm loss: 4.326500E+00 | loss scale: 8192.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.848 | TFLOPs: 71.64 | +[default7]: iteration 766/ 3814 | consumed samples: 392192 | consumed tokens: 803209216 | elapsed time per iteration (s): 5.41 | learning rate: 9.694E-04 | global batch size: 512 | lm loss: 4.308076E+00 | loss scale: 8192.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 767/ 3814 | consumed samples: 392704 | consumed tokens: 804257792 | elapsed time per iteration (s): 5.41 | learning rate: 9.692E-04 | global batch size: 512 | lm loss: 4.315421E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.719 | TFLOPs: 71.54 | +[default7]: iteration 768/ 3814 | consumed samples: 393216 | consumed tokens: 805306368 | elapsed time per iteration (s): 5.41 | learning rate: 9.691E-04 | global batch size: 512 | lm loss: 4.304098E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 769/ 3814 | consumed samples: 393728 | consumed tokens: 806354944 | elapsed time per iteration (s): 5.41 | learning rate: 9.689E-04 | global batch size: 512 | lm loss: 4.285424E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.629 | TFLOPs: 71.47 | +[default7]: iteration 770/ 3814 | consumed samples: 394240 | consumed tokens: 807403520 | elapsed time per iteration (s): 5.40 | learning rate: 9.688E-04 | global batch size: 512 | lm loss: 4.301469E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.733 | TFLOPs: 71.55 | +[default7]: iteration 771/ 3814 | consumed samples: 394752 | consumed tokens: 808452096 | elapsed time per iteration (s): 5.40 | learning rate: 9.686E-04 | global batch size: 512 | lm loss: 4.277471E+00 | loss scale: 8192.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.772 | TFLOPs: 71.58 | +[default7]: iteration 772/ 3814 | consumed samples: 395264 | consumed tokens: 809500672 | elapsed time per iteration (s): 5.40 | learning rate: 9.685E-04 | global batch size: 512 | lm loss: 4.296800E+00 | loss scale: 8192.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.867 | TFLOPs: 71.65 | +[default7]: iteration 773/ 3814 | consumed samples: 395776 | consumed tokens: 810549248 | elapsed time per iteration (s): 5.40 | learning rate: 9.683E-04 | global batch size: 512 | lm loss: 4.282505E+00 | loss scale: 8192.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.833 | TFLOPs: 71.63 | +[default7]: iteration 774/ 3814 | consumed samples: 396288 | consumed tokens: 811597824 | elapsed time per iteration (s): 5.38 | learning rate: 9.682E-04 | global batch size: 512 | lm loss: 4.288601E+00 | loss scale: 8192.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.175 | TFLOPs: 71.89 | +[default7]: iteration 775/ 3814 | consumed samples: 396800 | consumed tokens: 812646400 | elapsed time per iteration (s): 5.41 | learning rate: 9.680E-04 | global batch size: 512 | lm loss: 4.299775E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.722 | TFLOPs: 71.54 | +[default7]: iteration 776/ 3814 | consumed samples: 397312 | consumed tokens: 813694976 | elapsed time per iteration (s): 5.42 | learning rate: 9.679E-04 | global batch size: 512 | lm loss: 4.285431E+00 | loss scale: 8192.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 777/ 3814 | consumed samples: 397824 | consumed tokens: 814743552 | elapsed time per iteration (s): 5.39 | learning rate: 9.677E-04 | global batch size: 512 | lm loss: 4.309164E+00 | loss scale: 8192.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.025 | TFLOPs: 71.77 | +[default7]: iteration 778/ 3814 | consumed samples: 398336 | consumed tokens: 815792128 | elapsed time per iteration (s): 5.38 | learning rate: 9.676E-04 | global batch size: 512 | lm loss: 4.251371E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.186 | TFLOPs: 71.89 | +[default7]: iteration 779/ 3814 | consumed samples: 398848 | consumed tokens: 816840704 | elapsed time per iteration (s): 5.40 | learning rate: 9.674E-04 | global batch size: 512 | lm loss: 4.271130E+00 | loss scale: 8192.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.763 | TFLOPs: 71.58 | +[default7]: iteration 780/ 3814 | consumed samples: 399360 | consumed tokens: 817889280 | elapsed time per iteration (s): 5.38 | learning rate: 9.673E-04 | global batch size: 512 | lm loss: 4.245711E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.142 | TFLOPs: 71.86 | +[default7]: iteration 781/ 3814 | consumed samples: 399872 | consumed tokens: 818937856 | elapsed time per iteration (s): 5.40 | learning rate: 9.671E-04 | global batch size: 512 | lm loss: 4.274801E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.769 | TFLOPs: 71.58 | +[default7]: iteration 782/ 3814 | consumed samples: 400384 | consumed tokens: 819986432 | elapsed time per iteration (s): 5.42 | learning rate: 9.669E-04 | global batch size: 512 | lm loss: 4.235840E+00 | loss scale: 8192.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.394 | TFLOPs: 71.30 | +[default7]: iteration 783/ 3814 | consumed samples: 400896 | consumed tokens: 821035008 | elapsed time per iteration (s): 5.37 | learning rate: 9.668E-04 | global batch size: 512 | lm loss: 4.227991E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.307 | TFLOPs: 71.99 | +[default7]: iteration 784/ 3814 | consumed samples: 401408 | consumed tokens: 822083584 | elapsed time per iteration (s): 5.37 | learning rate: 9.666E-04 | global batch size: 512 | lm loss: 4.229736E+00 | loss scale: 8192.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.304 | TFLOPs: 71.98 | +[default7]: iteration 785/ 3814 | consumed samples: 401920 | consumed tokens: 823132160 | elapsed time per iteration (s): 5.40 | learning rate: 9.665E-04 | global batch size: 512 | lm loss: 4.247071E+00 | loss scale: 8192.0 | grad norm: 0.161 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.809 | TFLOPs: 71.61 | +[default7]: iteration 786/ 3814 | consumed samples: 402432 | consumed tokens: 824180736 | elapsed time per iteration (s): 5.42 | learning rate: 9.663E-04 | global batch size: 512 | lm loss: 4.272850E+00 | loss scale: 8192.0 | grad norm: 0.143 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.410 | TFLOPs: 71.31 | +[default7]: iteration 787/ 3814 | consumed samples: 402944 | consumed tokens: 825229312 | elapsed time per iteration (s): 5.41 | learning rate: 9.662E-04 | global batch size: 512 | lm loss: 4.273137E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.616 | TFLOPs: 71.46 | +[default7]: iteration 788/ 3814 | consumed samples: 403456 | consumed tokens: 826277888 | elapsed time per iteration (s): 5.38 | learning rate: 9.660E-04 | global batch size: 512 | lm loss: 4.251527E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.188 | TFLOPs: 71.90 | +[default7]: iteration 789/ 3814 | consumed samples: 403968 | consumed tokens: 827326464 | elapsed time per iteration (s): 5.39 | learning rate: 9.659E-04 | global batch size: 512 | lm loss: 4.251815E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.918 | TFLOPs: 71.69 | +[default7]: iteration 790/ 3814 | consumed samples: 404480 | consumed tokens: 828375040 | elapsed time per iteration (s): 5.41 | learning rate: 9.657E-04 | global batch size: 512 | lm loss: 4.233918E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.555 | TFLOPs: 71.42 | +[default7]: iteration 791/ 3814 | consumed samples: 404992 | consumed tokens: 829423616 | elapsed time per iteration (s): 5.42 | learning rate: 9.656E-04 | global batch size: 512 | lm loss: 4.222433E+00 | loss scale: 8192.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.506 | TFLOPs: 71.38 | +[default7]: iteration 792/ 3814 | consumed samples: 405504 | consumed tokens: 830472192 | elapsed time per iteration (s): 5.42 | learning rate: 9.654E-04 | global batch size: 512 | lm loss: 4.258236E+00 | loss scale: 8192.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 793/ 3814 | consumed samples: 406016 | consumed tokens: 831520768 | elapsed time per iteration (s): 5.42 | learning rate: 9.652E-04 | global batch size: 512 | lm loss: 4.231897E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.499 | TFLOPs: 71.38 | +[default7]: iteration 794/ 3814 | consumed samples: 406528 | consumed tokens: 832569344 | elapsed time per iteration (s): 5.41 | learning rate: 9.651E-04 | global batch size: 512 | lm loss: 4.226698E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 795/ 3814 | consumed samples: 407040 | consumed tokens: 833617920 | elapsed time per iteration (s): 5.42 | learning rate: 9.649E-04 | global batch size: 512 | lm loss: 4.219944E+00 | loss scale: 8192.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.443 | TFLOPs: 71.33 | +[default7]: iteration 796/ 3814 | consumed samples: 407552 | consumed tokens: 834666496 | elapsed time per iteration (s): 5.42 | learning rate: 9.648E-04 | global batch size: 512 | lm loss: 4.213585E+00 | loss scale: 8192.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.411 | TFLOPs: 71.31 | +[default7]: iteration 797/ 3814 | consumed samples: 408064 | consumed tokens: 835715072 | elapsed time per iteration (s): 5.42 | learning rate: 9.646E-04 | global batch size: 512 | lm loss: 4.208957E+00 | loss scale: 8192.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.502 | TFLOPs: 71.38 | +[default7]: iteration 798/ 3814 | consumed samples: 408576 | consumed tokens: 836763648 | elapsed time per iteration (s): 5.43 | learning rate: 9.644E-04 | global batch size: 512 | lm loss: 4.191415E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 799/ 3814 | consumed samples: 409088 | consumed tokens: 837812224 | elapsed time per iteration (s): 5.42 | learning rate: 9.643E-04 | global batch size: 512 | lm loss: 4.207687E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.488 | TFLOPs: 71.37 | +[default7]: iteration 800/ 3814 | consumed samples: 409600 | consumed tokens: 838860800 | elapsed time per iteration (s): 5.42 | learning rate: 9.641E-04 | global batch size: 512 | lm loss: 4.187036E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.380 | TFLOPs: 71.29 | +[default7]: iteration 801/ 3814 | consumed samples: 410112 | consumed tokens: 839909376 | elapsed time per iteration (s): 5.41 | learning rate: 9.640E-04 | global batch size: 512 | lm loss: 4.207813E+00 | loss scale: 8192.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.582 | TFLOPs: 71.44 | +[default7]: iteration 802/ 3814 | consumed samples: 410624 | consumed tokens: 840957952 | elapsed time per iteration (s): 5.43 | learning rate: 9.638E-04 | global batch size: 512 | lm loss: 4.188968E+00 | loss scale: 8192.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.306 | TFLOPs: 71.23 | +[default7]: iteration 803/ 3814 | consumed samples: 411136 | consumed tokens: 842006528 | elapsed time per iteration (s): 5.42 | learning rate: 9.636E-04 | global batch size: 512 | lm loss: 4.228027E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 804/ 3814 | consumed samples: 411648 | consumed tokens: 843055104 | elapsed time per iteration (s): 5.42 | learning rate: 9.635E-04 | global batch size: 512 | lm loss: 4.197236E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 805/ 3814 | consumed samples: 412160 | consumed tokens: 844103680 | elapsed time per iteration (s): 5.41 | learning rate: 9.633E-04 | global batch size: 512 | lm loss: 4.190719E+00 | loss scale: 8192.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.619 | TFLOPs: 71.47 | +[default7]: iteration 806/ 3814 | consumed samples: 412672 | consumed tokens: 845152256 | elapsed time per iteration (s): 5.40 | learning rate: 9.632E-04 | global batch size: 512 | lm loss: 4.204469E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.762 | TFLOPs: 71.57 | +[default7]: iteration 807/ 3814 | consumed samples: 413184 | consumed tokens: 846200832 | elapsed time per iteration (s): 5.40 | learning rate: 9.630E-04 | global batch size: 512 | lm loss: 4.188242E+00 | loss scale: 8192.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.831 | TFLOPs: 71.63 | +[default7]: iteration 808/ 3814 | consumed samples: 413696 | consumed tokens: 847249408 | elapsed time per iteration (s): 5.42 | learning rate: 9.628E-04 | global batch size: 512 | lm loss: 4.171384E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.388 | TFLOPs: 71.29 | +[default7]: iteration 809/ 3814 | consumed samples: 414208 | consumed tokens: 848297984 | elapsed time per iteration (s): 5.40 | learning rate: 9.627E-04 | global batch size: 512 | lm loss: 4.179982E+00 | loss scale: 8192.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.781 | TFLOPs: 71.59 | +[default7]: iteration 810/ 3814 | consumed samples: 414720 | consumed tokens: 849346560 | elapsed time per iteration (s): 5.40 | learning rate: 9.625E-04 | global batch size: 512 | lm loss: 4.183407E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.842 | TFLOPs: 71.64 | +[default7]: iteration 811/ 3814 | consumed samples: 415232 | consumed tokens: 850395136 | elapsed time per iteration (s): 5.42 | learning rate: 9.623E-04 | global batch size: 512 | lm loss: 4.153152E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 812/ 3814 | consumed samples: 415744 | consumed tokens: 851443712 | elapsed time per iteration (s): 5.41 | learning rate: 9.622E-04 | global batch size: 512 | lm loss: 4.167532E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.660 | TFLOPs: 71.50 | +[default7]: iteration 813/ 3814 | consumed samples: 416256 | consumed tokens: 852492288 | elapsed time per iteration (s): 5.42 | learning rate: 9.620E-04 | global batch size: 512 | lm loss: 4.148662E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.409 | TFLOPs: 71.31 | +[default7]: iteration 814/ 3814 | consumed samples: 416768 | consumed tokens: 853540864 | elapsed time per iteration (s): 5.41 | learning rate: 9.619E-04 | global batch size: 512 | lm loss: 4.168719E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.661 | TFLOPs: 71.50 | +[default7]: iteration 815/ 3814 | consumed samples: 417280 | consumed tokens: 854589440 | elapsed time per iteration (s): 5.41 | learning rate: 9.617E-04 | global batch size: 512 | lm loss: 4.182814E+00 | loss scale: 8192.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.727 | TFLOPs: 71.55 | +[default7]: iteration 816/ 3814 | consumed samples: 417792 | consumed tokens: 855638016 | elapsed time per iteration (s): 5.41 | learning rate: 9.615E-04 | global batch size: 512 | lm loss: 4.185331E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.558 | TFLOPs: 71.42 | +[default7]: iteration 817/ 3814 | consumed samples: 418304 | consumed tokens: 856686592 | elapsed time per iteration (s): 5.41 | learning rate: 9.614E-04 | global batch size: 512 | lm loss: 4.167561E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.666 | TFLOPs: 71.50 | +[default7]: iteration 818/ 3814 | consumed samples: 418816 | consumed tokens: 857735168 | elapsed time per iteration (s): 5.41 | learning rate: 9.612E-04 | global batch size: 512 | lm loss: 4.195529E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.627 | TFLOPs: 71.47 | +[default7]: iteration 819/ 3814 | consumed samples: 419328 | consumed tokens: 858783744 | elapsed time per iteration (s): 5.40 | learning rate: 9.610E-04 | global batch size: 512 | lm loss: 4.168196E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 820/ 3814 | consumed samples: 419840 | consumed tokens: 859832320 | elapsed time per iteration (s): 5.41 | learning rate: 9.609E-04 | global batch size: 512 | lm loss: 4.162785E+00 | loss scale: 8192.0 | grad norm: 0.139 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 821/ 3814 | consumed samples: 420352 | consumed tokens: 860880896 | elapsed time per iteration (s): 5.42 | learning rate: 9.607E-04 | global batch size: 512 | lm loss: 4.171123E+00 | loss scale: 8192.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 822/ 3814 | consumed samples: 420864 | consumed tokens: 861929472 | elapsed time per iteration (s): 5.43 | learning rate: 9.605E-04 | global batch size: 512 | lm loss: 4.188857E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.249 | TFLOPs: 71.19 | +[default7]: iteration 823/ 3814 | consumed samples: 421376 | consumed tokens: 862978048 | elapsed time per iteration (s): 5.43 | learning rate: 9.604E-04 | global batch size: 512 | lm loss: 4.147352E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.363 | TFLOPs: 71.27 | +[default7]: iteration 824/ 3814 | consumed samples: 421888 | consumed tokens: 864026624 | elapsed time per iteration (s): 5.40 | learning rate: 9.602E-04 | global batch size: 512 | lm loss: 4.146285E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.801 | TFLOPs: 71.60 | +[default7]: iteration 825/ 3814 | consumed samples: 422400 | consumed tokens: 865075200 | elapsed time per iteration (s): 5.40 | learning rate: 9.600E-04 | global batch size: 512 | lm loss: 4.165767E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.760 | TFLOPs: 71.57 | +[default7]: iteration 826/ 3814 | consumed samples: 422912 | consumed tokens: 866123776 | elapsed time per iteration (s): 5.39 | learning rate: 9.599E-04 | global batch size: 512 | lm loss: 4.156534E+00 | loss scale: 8192.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.042 | TFLOPs: 71.79 | +[default7]: iteration 827/ 3814 | consumed samples: 423424 | consumed tokens: 867172352 | elapsed time per iteration (s): 5.40 | learning rate: 9.597E-04 | global batch size: 512 | lm loss: 4.171617E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.794 | TFLOPs: 71.60 | +[default7]: iteration 828/ 3814 | consumed samples: 423936 | consumed tokens: 868220928 | elapsed time per iteration (s): 5.41 | learning rate: 9.595E-04 | global batch size: 512 | lm loss: 4.182305E+00 | loss scale: 8192.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.560 | TFLOPs: 71.42 | +[default7]: iteration 829/ 3814 | consumed samples: 424448 | consumed tokens: 869269504 | elapsed time per iteration (s): 5.40 | learning rate: 9.593E-04 | global batch size: 512 | lm loss: 4.150391E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.778 | TFLOPs: 71.59 | +[default7]: iteration 830/ 3814 | consumed samples: 424960 | consumed tokens: 870318080 | elapsed time per iteration (s): 5.39 | learning rate: 9.592E-04 | global batch size: 512 | lm loss: 4.153550E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.030 | TFLOPs: 71.78 | +[default7]: iteration 831/ 3814 | consumed samples: 425472 | consumed tokens: 871366656 | elapsed time per iteration (s): 5.39 | learning rate: 9.590E-04 | global batch size: 512 | lm loss: 4.115952E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.050 | TFLOPs: 71.79 | +[default7]: iteration 832/ 3814 | consumed samples: 425984 | consumed tokens: 872415232 | elapsed time per iteration (s): 5.46 | learning rate: 9.588E-04 | global batch size: 512 | lm loss: 4.136443E+00 | loss scale: 8192.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.846 | TFLOPs: 70.88 | +[default7]: iteration 833/ 3814 | consumed samples: 426496 | consumed tokens: 873463808 | elapsed time per iteration (s): 5.41 | learning rate: 9.587E-04 | global batch size: 512 | lm loss: 4.150321E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.681 | TFLOPs: 71.51 | +[default7]: iteration 834/ 3814 | consumed samples: 427008 | consumed tokens: 874512384 | elapsed time per iteration (s): 5.41 | learning rate: 9.585E-04 | global batch size: 512 | lm loss: 4.147053E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.47 | +[default7]: iteration 835/ 3814 | consumed samples: 427520 | consumed tokens: 875560960 | elapsed time per iteration (s): 5.41 | learning rate: 9.583E-04 | global batch size: 512 | lm loss: 4.083259E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.692 | TFLOPs: 71.52 | +[default7]: iteration 836/ 3814 | consumed samples: 428032 | consumed tokens: 876609536 | elapsed time per iteration (s): 5.40 | learning rate: 9.581E-04 | global batch size: 512 | lm loss: 4.115165E+00 | loss scale: 8192.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.826 | TFLOPs: 71.62 | +[default7]: iteration 837/ 3814 | consumed samples: 428544 | consumed tokens: 877658112 | elapsed time per iteration (s): 5.39 | learning rate: 9.580E-04 | global batch size: 512 | lm loss: 4.123104E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.912 | TFLOPs: 71.69 | +[default7]: iteration 838/ 3814 | consumed samples: 429056 | consumed tokens: 878706688 | elapsed time per iteration (s): 5.40 | learning rate: 9.578E-04 | global batch size: 512 | lm loss: 4.117439E+00 | loss scale: 8192.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.841 | TFLOPs: 71.63 | +[default7]: iteration 839/ 3814 | consumed samples: 429568 | consumed tokens: 879755264 | elapsed time per iteration (s): 5.41 | learning rate: 9.576E-04 | global batch size: 512 | lm loss: 4.131541E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 840/ 3814 | consumed samples: 430080 | consumed tokens: 880803840 | elapsed time per iteration (s): 5.43 | learning rate: 9.575E-04 | global batch size: 512 | lm loss: 4.153267E+00 | loss scale: 8192.0 | grad norm: 0.146 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.360 | TFLOPs: 71.27 | +[default7]: iteration 841/ 3814 | consumed samples: 430592 | consumed tokens: 881852416 | elapsed time per iteration (s): 5.44 | learning rate: 9.573E-04 | global batch size: 512 | lm loss: 4.142547E+00 | loss scale: 8192.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.182 | TFLOPs: 71.14 | +[default7]: iteration 842/ 3814 | consumed samples: 431104 | consumed tokens: 882900992 | elapsed time per iteration (s): 5.42 | learning rate: 9.571E-04 | global batch size: 512 | lm loss: 4.152344E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.439 | TFLOPs: 71.33 | +[default7]: iteration 843/ 3814 | consumed samples: 431616 | consumed tokens: 883949568 | elapsed time per iteration (s): 5.42 | learning rate: 9.569E-04 | global batch size: 512 | lm loss: 4.128252E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.431 | TFLOPs: 71.33 | +[default7]: iteration 844/ 3814 | consumed samples: 432128 | consumed tokens: 884998144 | elapsed time per iteration (s): 5.42 | learning rate: 9.568E-04 | global batch size: 512 | lm loss: 4.116621E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.418 | TFLOPs: 71.31 | +[default7]: iteration 845/ 3814 | consumed samples: 432640 | consumed tokens: 886046720 | elapsed time per iteration (s): 5.42 | learning rate: 9.566E-04 | global batch size: 512 | lm loss: 4.139468E+00 | loss scale: 8192.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.500 | TFLOPs: 71.38 | +[default7]: iteration 846/ 3814 | consumed samples: 433152 | consumed tokens: 887095296 | elapsed time per iteration (s): 5.41 | learning rate: 9.564E-04 | global batch size: 512 | lm loss: 4.133089E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.590 | TFLOPs: 71.44 | +[default7]: iteration 847/ 3814 | consumed samples: 433664 | consumed tokens: 888143872 | elapsed time per iteration (s): 5.42 | learning rate: 9.562E-04 | global batch size: 512 | lm loss: 4.096426E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.438 | TFLOPs: 71.33 | +[default7]: iteration 848/ 3814 | consumed samples: 434176 | consumed tokens: 889192448 | elapsed time per iteration (s): 5.42 | learning rate: 9.561E-04 | global batch size: 512 | lm loss: 4.112541E+00 | loss scale: 8192.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.543 | TFLOPs: 71.41 | +[default7]: iteration 849/ 3814 | consumed samples: 434688 | consumed tokens: 890241024 | elapsed time per iteration (s): 5.42 | learning rate: 9.559E-04 | global batch size: 512 | lm loss: 4.088303E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.511 | TFLOPs: 71.39 | +[default7]: iteration 850/ 3814 | consumed samples: 435200 | consumed tokens: 891289600 | elapsed time per iteration (s): 5.41 | learning rate: 9.557E-04 | global batch size: 512 | lm loss: 4.095063E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 851/ 3814 | consumed samples: 435712 | consumed tokens: 892338176 | elapsed time per iteration (s): 5.41 | learning rate: 9.555E-04 | global batch size: 512 | lm loss: 4.109057E+00 | loss scale: 8192.0 | grad norm: 0.065 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.684 | TFLOPs: 71.52 | +[default7]: iteration 852/ 3814 | consumed samples: 436224 | consumed tokens: 893386752 | elapsed time per iteration (s): 5.42 | learning rate: 9.553E-04 | global batch size: 512 | lm loss: 4.084609E+00 | loss scale: 8192.0 | grad norm: 0.067 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 853/ 3814 | consumed samples: 436736 | consumed tokens: 894435328 | elapsed time per iteration (s): 5.42 | learning rate: 9.552E-04 | global batch size: 512 | lm loss: 4.073705E+00 | loss scale: 8192.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.493 | TFLOPs: 71.37 | +[default7]: iteration 854/ 3814 | consumed samples: 437248 | consumed tokens: 895483904 | elapsed time per iteration (s): 5.39 | learning rate: 9.550E-04 | global batch size: 512 | lm loss: 4.038041E+00 | loss scale: 8192.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.999 | TFLOPs: 71.75 | +[default7]: iteration 855/ 3814 | consumed samples: 437760 | consumed tokens: 896532480 | elapsed time per iteration (s): 5.41 | learning rate: 9.548E-04 | global batch size: 512 | lm loss: 4.090663E+00 | loss scale: 8192.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.582 | TFLOPs: 71.44 | +[default7]: iteration 856/ 3814 | consumed samples: 438272 | consumed tokens: 897581056 | elapsed time per iteration (s): 5.43 | learning rate: 9.546E-04 | global batch size: 512 | lm loss: 4.104561E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.294 | TFLOPs: 71.22 | +[default7]: iteration 857/ 3814 | consumed samples: 438784 | consumed tokens: 898629632 | elapsed time per iteration (s): 5.44 | learning rate: 9.545E-04 | global batch size: 512 | lm loss: 4.051007E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.064 | TFLOPs: 71.05 | +[default7]: iteration 858/ 3814 | consumed samples: 439296 | consumed tokens: 899678208 | elapsed time per iteration (s): 5.45 | learning rate: 9.543E-04 | global batch size: 512 | lm loss: 4.083394E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.021 | TFLOPs: 71.02 | +[default7]: iteration 859/ 3814 | consumed samples: 439808 | consumed tokens: 900726784 | elapsed time per iteration (s): 5.42 | learning rate: 9.541E-04 | global batch size: 512 | lm loss: 4.090508E+00 | loss scale: 8192.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.495 | TFLOPs: 71.37 | +[default7]: iteration 860/ 3814 | consumed samples: 440320 | consumed tokens: 901775360 | elapsed time per iteration (s): 5.40 | learning rate: 9.539E-04 | global batch size: 512 | lm loss: 4.100019E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.816 | TFLOPs: 71.62 | +[default7]: iteration 861/ 3814 | consumed samples: 440832 | consumed tokens: 902823936 | elapsed time per iteration (s): 5.41 | learning rate: 9.537E-04 | global batch size: 512 | lm loss: 4.090946E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.647 | TFLOPs: 71.49 | +[default7]: iteration 862/ 3814 | consumed samples: 441344 | consumed tokens: 903872512 | elapsed time per iteration (s): 5.44 | learning rate: 9.536E-04 | global batch size: 512 | lm loss: 4.078511E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.129 | TFLOPs: 71.10 | +[default7]: iteration 863/ 3814 | consumed samples: 441856 | consumed tokens: 904921088 | elapsed time per iteration (s): 5.42 | learning rate: 9.534E-04 | global batch size: 512 | lm loss: 4.038083E+00 | loss scale: 8192.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 864/ 3814 | consumed samples: 442368 | consumed tokens: 905969664 | elapsed time per iteration (s): 5.40 | learning rate: 9.532E-04 | global batch size: 512 | lm loss: 4.069535E+00 | loss scale: 8192.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.840 | TFLOPs: 71.63 | +[default7]: iteration 865/ 3814 | consumed samples: 442880 | consumed tokens: 907018240 | elapsed time per iteration (s): 5.39 | learning rate: 9.530E-04 | global batch size: 512 | lm loss: 4.037474E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.911 | TFLOPs: 71.69 | +[default7]: iteration 866/ 3814 | consumed samples: 443392 | consumed tokens: 908066816 | elapsed time per iteration (s): 5.42 | learning rate: 9.528E-04 | global batch size: 512 | lm loss: 4.044816E+00 | loss scale: 8192.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.472 | TFLOPs: 71.36 | +[default7]: iteration 867/ 3814 | consumed samples: 443904 | consumed tokens: 909115392 | elapsed time per iteration (s): 5.44 | learning rate: 9.526E-04 | global batch size: 512 | lm loss: 4.038741E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.042 | TFLOPs: 71.03 | +[default7]: iteration 868/ 3814 | consumed samples: 444416 | consumed tokens: 910163968 | elapsed time per iteration (s): 5.40 | learning rate: 9.525E-04 | global batch size: 512 | lm loss: 4.031717E+00 | loss scale: 8192.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.828 | TFLOPs: 71.62 | +[default7]: iteration 869/ 3814 | consumed samples: 444928 | consumed tokens: 911212544 | elapsed time per iteration (s): 5.42 | learning rate: 9.523E-04 | global batch size: 512 | lm loss: 4.076289E+00 | loss scale: 8192.0 | grad norm: 0.147 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 870/ 3814 | consumed samples: 445440 | consumed tokens: 912261120 | elapsed time per iteration (s): 5.42 | learning rate: 9.521E-04 | global batch size: 512 | lm loss: 4.106956E+00 | loss scale: 8192.0 | grad norm: 0.158 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.462 | TFLOPs: 71.35 | +[default7]: iteration 871/ 3814 | consumed samples: 445952 | consumed tokens: 913309696 | elapsed time per iteration (s): 5.41 | learning rate: 9.519E-04 | global batch size: 512 | lm loss: 4.091662E+00 | loss scale: 8192.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.649 | TFLOPs: 71.49 | +[default7]: iteration 872/ 3814 | consumed samples: 446464 | consumed tokens: 914358272 | elapsed time per iteration (s): 5.42 | learning rate: 9.517E-04 | global batch size: 512 | lm loss: 4.097868E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.403 | TFLOPs: 71.30 | +[default7]: iteration 873/ 3814 | consumed samples: 446976 | consumed tokens: 915406848 | elapsed time per iteration (s): 5.41 | learning rate: 9.515E-04 | global batch size: 512 | lm loss: 4.099156E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 874/ 3814 | consumed samples: 447488 | consumed tokens: 916455424 | elapsed time per iteration (s): 5.41 | learning rate: 9.514E-04 | global batch size: 512 | lm loss: 4.067172E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.726 | TFLOPs: 71.55 | +[default7]: iteration 875/ 3814 | consumed samples: 448000 | consumed tokens: 917504000 | elapsed time per iteration (s): 5.41 | learning rate: 9.512E-04 | global batch size: 512 | lm loss: 4.042486E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 876/ 3814 | consumed samples: 448512 | consumed tokens: 918552576 | elapsed time per iteration (s): 5.41 | learning rate: 9.510E-04 | global batch size: 512 | lm loss: 4.045116E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 877/ 3814 | consumed samples: 449024 | consumed tokens: 919601152 | elapsed time per iteration (s): 5.43 | learning rate: 9.508E-04 | global batch size: 512 | lm loss: 4.073288E+00 | loss scale: 8192.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.213 | TFLOPs: 71.16 | +[default7]: iteration 878/ 3814 | consumed samples: 449536 | consumed tokens: 920649728 | elapsed time per iteration (s): 5.41 | learning rate: 9.506E-04 | global batch size: 512 | lm loss: 4.048798E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.583 | TFLOPs: 71.44 | +[default7]: iteration 879/ 3814 | consumed samples: 450048 | consumed tokens: 921698304 | elapsed time per iteration (s): 5.42 | learning rate: 9.504E-04 | global batch size: 512 | lm loss: 4.036306E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.422 | TFLOPs: 71.32 | +[default7]: iteration 880/ 3814 | consumed samples: 450560 | consumed tokens: 922746880 | elapsed time per iteration (s): 5.43 | learning rate: 9.502E-04 | global batch size: 512 | lm loss: 4.011985E+00 | loss scale: 8192.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.353 | TFLOPs: 71.27 | +[default7]: iteration 881/ 3814 | consumed samples: 451072 | consumed tokens: 923795456 | elapsed time per iteration (s): 5.42 | learning rate: 9.501E-04 | global batch size: 512 | lm loss: 4.025365E+00 | loss scale: 8192.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.466 | TFLOPs: 71.35 | +[default7]: iteration 882/ 3814 | consumed samples: 451584 | consumed tokens: 924844032 | elapsed time per iteration (s): 5.42 | learning rate: 9.499E-04 | global batch size: 512 | lm loss: 4.032502E+00 | loss scale: 8192.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.395 | TFLOPs: 71.30 | +[default7]: iteration 883/ 3814 | consumed samples: 452096 | consumed tokens: 925892608 | elapsed time per iteration (s): 5.42 | learning rate: 9.497E-04 | global batch size: 512 | lm loss: 3.978524E+00 | loss scale: 8192.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.399 | TFLOPs: 71.30 | +[default7]: iteration 884/ 3814 | consumed samples: 452608 | consumed tokens: 926941184 | elapsed time per iteration (s): 5.43 | learning rate: 9.495E-04 | global batch size: 512 | lm loss: 4.026020E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 885/ 3814 | consumed samples: 453120 | consumed tokens: 927989760 | elapsed time per iteration (s): 5.41 | learning rate: 9.493E-04 | global batch size: 512 | lm loss: 4.048073E+00 | loss scale: 8192.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 886/ 3814 | consumed samples: 453632 | consumed tokens: 929038336 | elapsed time per iteration (s): 5.41 | learning rate: 9.491E-04 | global batch size: 512 | lm loss: 4.024578E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.615 | TFLOPs: 71.46 | +[default7]: iteration 887/ 3814 | consumed samples: 454144 | consumed tokens: 930086912 | elapsed time per iteration (s): 5.40 | learning rate: 9.489E-04 | global batch size: 512 | lm loss: 3.970982E+00 | loss scale: 8192.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.746 | TFLOPs: 71.56 | +[default7]: iteration 888/ 3814 | consumed samples: 454656 | consumed tokens: 931135488 | elapsed time per iteration (s): 5.42 | learning rate: 9.487E-04 | global batch size: 512 | lm loss: 4.017374E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.401 | TFLOPs: 71.30 | +[default7]: iteration 889/ 3814 | consumed samples: 455168 | consumed tokens: 932184064 | elapsed time per iteration (s): 5.39 | learning rate: 9.485E-04 | global batch size: 512 | lm loss: 4.015644E+00 | loss scale: 8192.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.914 | TFLOPs: 71.69 | +[default7]: iteration 890/ 3814 | consumed samples: 455680 | consumed tokens: 933232640 | elapsed time per iteration (s): 5.42 | learning rate: 9.484E-04 | global batch size: 512 | lm loss: 4.012274E+00 | loss scale: 8192.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.466 | TFLOPs: 71.35 | +[default7]: iteration 891/ 3814 | consumed samples: 456192 | consumed tokens: 934281216 | elapsed time per iteration (s): 5.42 | learning rate: 9.482E-04 | global batch size: 512 | lm loss: 4.015715E+00 | loss scale: 8192.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.491 | TFLOPs: 71.37 | +[default7]: iteration 892/ 3814 | consumed samples: 456704 | consumed tokens: 935329792 | elapsed time per iteration (s): 5.41 | learning rate: 9.480E-04 | global batch size: 512 | lm loss: 4.006466E+00 | loss scale: 8192.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.726 | TFLOPs: 71.55 | +[default7]: iteration 893/ 3814 | consumed samples: 457216 | consumed tokens: 936378368 | elapsed time per iteration (s): 5.40 | learning rate: 9.478E-04 | global batch size: 512 | lm loss: 3.990160E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.822 | TFLOPs: 71.62 | +[default7]: iteration 894/ 3814 | consumed samples: 457728 | consumed tokens: 937426944 | elapsed time per iteration (s): 5.42 | learning rate: 9.476E-04 | global batch size: 512 | lm loss: 4.019519E+00 | loss scale: 8192.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.500 | TFLOPs: 71.38 | +[default7]: iteration 895/ 3814 | consumed samples: 458240 | consumed tokens: 938475520 | elapsed time per iteration (s): 5.43 | learning rate: 9.474E-04 | global batch size: 512 | lm loss: 4.036668E+00 | loss scale: 8192.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.254 | TFLOPs: 71.19 | +[default7]: iteration 896/ 3814 | consumed samples: 458752 | consumed tokens: 939524096 | elapsed time per iteration (s): 5.42 | learning rate: 9.472E-04 | global batch size: 512 | lm loss: 4.022719E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 897/ 3814 | consumed samples: 459264 | consumed tokens: 940572672 | elapsed time per iteration (s): 5.43 | learning rate: 9.470E-04 | global batch size: 512 | lm loss: 3.999572E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 898/ 3814 | consumed samples: 459776 | consumed tokens: 941621248 | elapsed time per iteration (s): 5.40 | learning rate: 9.468E-04 | global batch size: 512 | lm loss: 3.963806E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.734 | TFLOPs: 71.55 | +[default7]: iteration 899/ 3814 | consumed samples: 460288 | consumed tokens: 942669824 | elapsed time per iteration (s): 5.43 | learning rate: 9.466E-04 | global batch size: 512 | lm loss: 3.998171E+00 | loss scale: 8192.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.354 | TFLOPs: 71.27 | +[default7]: iteration 900/ 3814 | consumed samples: 460800 | consumed tokens: 943718400 | elapsed time per iteration (s): 5.42 | learning rate: 9.464E-04 | global batch size: 512 | lm loss: 4.002934E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.522 | TFLOPs: 71.39 | +[default7]: iteration 901/ 3814 | consumed samples: 461312 | consumed tokens: 944766976 | elapsed time per iteration (s): 5.42 | learning rate: 9.462E-04 | global batch size: 512 | lm loss: 4.007532E+00 | loss scale: 8192.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.392 | TFLOPs: 71.30 | +[default7]: iteration 902/ 3814 | consumed samples: 461824 | consumed tokens: 945815552 | elapsed time per iteration (s): 5.42 | learning rate: 9.461E-04 | global batch size: 512 | lm loss: 3.979761E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 903/ 3814 | consumed samples: 462336 | consumed tokens: 946864128 | elapsed time per iteration (s): 5.41 | learning rate: 9.459E-04 | global batch size: 512 | lm loss: 3.989794E+00 | loss scale: 8192.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.702 | TFLOPs: 71.53 | +[default7]: iteration 904/ 3814 | consumed samples: 462848 | consumed tokens: 947912704 | elapsed time per iteration (s): 5.41 | learning rate: 9.457E-04 | global batch size: 512 | lm loss: 3.977045E+00 | loss scale: 8192.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.651 | TFLOPs: 71.49 | +[default7]: iteration 905/ 3814 | consumed samples: 463360 | consumed tokens: 948961280 | elapsed time per iteration (s): 5.43 | learning rate: 9.455E-04 | global batch size: 512 | lm loss: 4.003193E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.250 | TFLOPs: 71.19 | +[default7]: iteration 906/ 3814 | consumed samples: 463872 | consumed tokens: 950009856 | elapsed time per iteration (s): 5.42 | learning rate: 9.453E-04 | global batch size: 512 | lm loss: 4.002050E+00 | loss scale: 8192.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.454 | TFLOPs: 71.34 | +[default7]: iteration 907/ 3814 | consumed samples: 464384 | consumed tokens: 951058432 | elapsed time per iteration (s): 5.40 | learning rate: 9.451E-04 | global batch size: 512 | lm loss: 3.998704E+00 | loss scale: 8192.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.784 | TFLOPs: 71.59 | +[default7]: iteration 908/ 3814 | consumed samples: 464896 | consumed tokens: 952107008 | elapsed time per iteration (s): 5.44 | learning rate: 9.449E-04 | global batch size: 512 | lm loss: 3.988362E+00 | loss scale: 8192.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.177 | TFLOPs: 71.13 | +[default7]: iteration 909/ 3814 | consumed samples: 465408 | consumed tokens: 953155584 | elapsed time per iteration (s): 5.43 | learning rate: 9.447E-04 | global batch size: 512 | lm loss: 4.015204E+00 | loss scale: 8192.0 | grad norm: 0.155 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.375 | TFLOPs: 71.28 | +[default7]: iteration 910/ 3814 | consumed samples: 465920 | consumed tokens: 954204160 | elapsed time per iteration (s): 5.41 | learning rate: 9.445E-04 | global batch size: 512 | lm loss: 4.008695E+00 | loss scale: 8192.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.700 | TFLOPs: 71.53 | +[default7]: iteration 911/ 3814 | consumed samples: 466432 | consumed tokens: 955252736 | elapsed time per iteration (s): 5.41 | learning rate: 9.443E-04 | global batch size: 512 | lm loss: 3.961555E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.704 | TFLOPs: 71.53 | +[default7]: iteration 912/ 3814 | consumed samples: 466944 | consumed tokens: 956301312 | elapsed time per iteration (s): 5.40 | learning rate: 9.441E-04 | global batch size: 512 | lm loss: 4.002902E+00 | loss scale: 8192.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.768 | TFLOPs: 71.58 | +[default7]: iteration 913/ 3814 | consumed samples: 467456 | consumed tokens: 957349888 | elapsed time per iteration (s): 5.41 | learning rate: 9.439E-04 | global batch size: 512 | lm loss: 3.978185E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.637 | TFLOPs: 71.48 | +[default7]: iteration 914/ 3814 | consumed samples: 467968 | consumed tokens: 958398464 | elapsed time per iteration (s): 5.42 | learning rate: 9.437E-04 | global batch size: 512 | lm loss: 3.993969E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.486 | TFLOPs: 71.37 | +[default7]: iteration 915/ 3814 | consumed samples: 468480 | consumed tokens: 959447040 | elapsed time per iteration (s): 5.41 | learning rate: 9.435E-04 | global batch size: 512 | lm loss: 4.013032E+00 | loss scale: 8192.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default7]: iteration 916/ 3814 | consumed samples: 468992 | consumed tokens: 960495616 | elapsed time per iteration (s): 5.41 | learning rate: 9.433E-04 | global batch size: 512 | lm loss: 4.008237E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.609 | TFLOPs: 71.46 | +[default7]: iteration 917/ 3814 | consumed samples: 469504 | consumed tokens: 961544192 | elapsed time per iteration (s): 5.41 | learning rate: 9.431E-04 | global batch size: 512 | lm loss: 3.984762E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.576 | TFLOPs: 71.43 | +[default7]: iteration 918/ 3814 | consumed samples: 470016 | consumed tokens: 962592768 | elapsed time per iteration (s): 5.42 | learning rate: 9.429E-04 | global batch size: 512 | lm loss: 4.003749E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.485 | TFLOPs: 71.37 | +[default7]: iteration 919/ 3814 | consumed samples: 470528 | consumed tokens: 963641344 | elapsed time per iteration (s): 5.42 | learning rate: 9.427E-04 | global batch size: 512 | lm loss: 3.960893E+00 | loss scale: 8192.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.391 | TFLOPs: 71.29 | +[default7]: iteration 920/ 3814 | consumed samples: 471040 | consumed tokens: 964689920 | elapsed time per iteration (s): 5.43 | learning rate: 9.425E-04 | global batch size: 512 | lm loss: 3.962589E+00 | loss scale: 8192.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.303 | TFLOPs: 71.23 | +[default7]: iteration 921/ 3814 | consumed samples: 471552 | consumed tokens: 965738496 | elapsed time per iteration (s): 5.42 | learning rate: 9.423E-04 | global batch size: 512 | lm loss: 3.953394E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.481 | TFLOPs: 71.36 | +[default7]: iteration 922/ 3814 | consumed samples: 472064 | consumed tokens: 966787072 | elapsed time per iteration (s): 5.45 | learning rate: 9.421E-04 | global batch size: 512 | lm loss: 3.948019E+00 | loss scale: 8192.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.895 | TFLOPs: 70.92 | +[default7]: iteration 923/ 3814 | consumed samples: 472576 | consumed tokens: 967835648 | elapsed time per iteration (s): 5.43 | learning rate: 9.419E-04 | global batch size: 512 | lm loss: 3.972498E+00 | loss scale: 8192.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.245 | TFLOPs: 71.18 | +[default7]: iteration 924/ 3814 | consumed samples: 473088 | consumed tokens: 968884224 | elapsed time per iteration (s): 5.42 | learning rate: 9.417E-04 | global batch size: 512 | lm loss: 3.950088E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.536 | TFLOPs: 71.40 | +[default7]: iteration 925/ 3814 | consumed samples: 473600 | consumed tokens: 969932800 | elapsed time per iteration (s): 5.41 | learning rate: 9.415E-04 | global batch size: 512 | lm loss: 3.949098E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.622 | TFLOPs: 71.47 | +[default7]: iteration 926/ 3814 | consumed samples: 474112 | consumed tokens: 970981376 | elapsed time per iteration (s): 5.44 | learning rate: 9.413E-04 | global batch size: 512 | lm loss: 3.916520E+00 | loss scale: 8192.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.105 | TFLOPs: 71.08 | +[default7]: iteration 927/ 3814 | consumed samples: 474624 | consumed tokens: 972029952 | elapsed time per iteration (s): 5.40 | learning rate: 9.411E-04 | global batch size: 512 | lm loss: 3.920674E+00 | loss scale: 8192.0 | grad norm: 0.064 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.763 | TFLOPs: 71.58 | +[default7]: iteration 928/ 3814 | consumed samples: 475136 | consumed tokens: 973078528 | elapsed time per iteration (s): 5.42 | learning rate: 9.409E-04 | global batch size: 512 | lm loss: 3.952742E+00 | loss scale: 8192.0 | grad norm: 0.065 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 929/ 3814 | consumed samples: 475648 | consumed tokens: 974127104 | elapsed time per iteration (s): 5.43 | learning rate: 9.407E-04 | global batch size: 512 | lm loss: 3.946093E+00 | loss scale: 8192.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.270 | TFLOPs: 71.20 | +[default7]: iteration 930/ 3814 | consumed samples: 476160 | consumed tokens: 975175680 | elapsed time per iteration (s): 5.42 | learning rate: 9.405E-04 | global batch size: 512 | lm loss: 3.939451E+00 | loss scale: 8192.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 931/ 3814 | consumed samples: 476672 | consumed tokens: 976224256 | elapsed time per iteration (s): 5.41 | learning rate: 9.403E-04 | global batch size: 512 | lm loss: 3.916543E+00 | loss scale: 8192.0 | grad norm: 0.066 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.561 | TFLOPs: 71.42 | +[default7]: iteration 932/ 3814 | consumed samples: 477184 | consumed tokens: 977272832 | elapsed time per iteration (s): 5.43 | learning rate: 9.401E-04 | global batch size: 512 | lm loss: 3.936047E+00 | loss scale: 8192.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.228 | TFLOPs: 71.17 | +[default7]: iteration 933/ 3814 | consumed samples: 477696 | consumed tokens: 978321408 | elapsed time per iteration (s): 5.41 | learning rate: 9.399E-04 | global batch size: 512 | lm loss: 3.920913E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.612 | TFLOPs: 71.46 | +[default7]: iteration 934/ 3814 | consumed samples: 478208 | consumed tokens: 979369984 | elapsed time per iteration (s): 5.42 | learning rate: 9.397E-04 | global batch size: 512 | lm loss: 3.890194E+00 | loss scale: 8192.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.402 | TFLOPs: 71.30 | +[default7]: iteration 935/ 3814 | consumed samples: 478720 | consumed tokens: 980418560 | elapsed time per iteration (s): 5.42 | learning rate: 9.395E-04 | global batch size: 512 | lm loss: 3.936220E+00 | loss scale: 8192.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.491 | TFLOPs: 71.37 | +[default7]: iteration 936/ 3814 | consumed samples: 479232 | consumed tokens: 981467136 | elapsed time per iteration (s): 5.42 | learning rate: 9.393E-04 | global batch size: 512 | lm loss: 3.930486E+00 | loss scale: 8192.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 937/ 3814 | consumed samples: 479744 | consumed tokens: 982515712 | elapsed time per iteration (s): 5.42 | learning rate: 9.391E-04 | global batch size: 512 | lm loss: 3.971486E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.535 | TFLOPs: 71.40 | +[default7]: iteration 938/ 3814 | consumed samples: 480256 | consumed tokens: 983564288 | elapsed time per iteration (s): 5.40 | learning rate: 9.389E-04 | global batch size: 512 | lm loss: 3.925364E+00 | loss scale: 8192.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 939/ 3814 | consumed samples: 480768 | consumed tokens: 984612864 | elapsed time per iteration (s): 5.43 | learning rate: 9.386E-04 | global batch size: 512 | lm loss: 3.936160E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.265 | TFLOPs: 71.20 | +[default7]: iteration 940/ 3814 | consumed samples: 481280 | consumed tokens: 985661440 | elapsed time per iteration (s): 5.42 | learning rate: 9.384E-04 | global batch size: 512 | lm loss: 3.954588E+00 | loss scale: 8192.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 941/ 3814 | consumed samples: 481792 | consumed tokens: 986710016 | elapsed time per iteration (s): 5.41 | learning rate: 9.382E-04 | global batch size: 512 | lm loss: 3.951669E+00 | loss scale: 8192.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.600 | TFLOPs: 71.45 | +[default7]: iteration 942/ 3814 | consumed samples: 482304 | consumed tokens: 987758592 | elapsed time per iteration (s): 5.41 | learning rate: 9.380E-04 | global batch size: 512 | lm loss: 3.912501E+00 | loss scale: 8192.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.637 | TFLOPs: 71.48 | +[default7]: iteration 943/ 3814 | consumed samples: 482816 | consumed tokens: 988807168 | elapsed time per iteration (s): 5.41 | learning rate: 9.378E-04 | global batch size: 512 | lm loss: 3.956793E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.581 | TFLOPs: 71.44 | +[default7]: iteration 944/ 3814 | consumed samples: 483328 | consumed tokens: 989855744 | elapsed time per iteration (s): 5.42 | learning rate: 9.376E-04 | global batch size: 512 | lm loss: 3.902248E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 945/ 3814 | consumed samples: 483840 | consumed tokens: 990904320 | elapsed time per iteration (s): 5.41 | learning rate: 9.374E-04 | global batch size: 512 | lm loss: 3.941481E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.582 | TFLOPs: 71.44 | +[default7]: iteration 946/ 3814 | consumed samples: 484352 | consumed tokens: 991952896 | elapsed time per iteration (s): 5.42 | learning rate: 9.372E-04 | global batch size: 512 | lm loss: 3.922094E+00 | loss scale: 8192.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 947/ 3814 | consumed samples: 484864 | consumed tokens: 993001472 | elapsed time per iteration (s): 5.40 | learning rate: 9.370E-04 | global batch size: 512 | lm loss: 3.926237E+00 | loss scale: 8192.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.828 | TFLOPs: 71.62 | +[default7]: iteration 948/ 3814 | consumed samples: 485376 | consumed tokens: 994050048 | elapsed time per iteration (s): 5.40 | learning rate: 9.368E-04 | global batch size: 512 | lm loss: 3.924911E+00 | loss scale: 8192.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.838 | TFLOPs: 71.63 | +[default7]: iteration 949/ 3814 | consumed samples: 485888 | consumed tokens: 995098624 | elapsed time per iteration (s): 5.39 | learning rate: 9.366E-04 | global batch size: 512 | lm loss: 3.932837E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.005 | TFLOPs: 71.76 | +[default7]: iteration 950/ 3814 | consumed samples: 486400 | consumed tokens: 996147200 | elapsed time per iteration (s): 5.41 | learning rate: 9.364E-04 | global batch size: 512 | lm loss: 3.957945E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.594 | TFLOPs: 71.45 | +[default7]: iteration 951/ 3814 | consumed samples: 486912 | consumed tokens: 997195776 | elapsed time per iteration (s): 5.39 | learning rate: 9.361E-04 | global batch size: 512 | lm loss: 3.922401E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.037 | TFLOPs: 71.78 | +[default7]: iteration 952/ 3814 | consumed samples: 487424 | consumed tokens: 998244352 | elapsed time per iteration (s): 5.40 | learning rate: 9.359E-04 | global batch size: 512 | lm loss: 3.946939E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.846 | TFLOPs: 71.64 | +[default7]: iteration 953/ 3814 | consumed samples: 487936 | consumed tokens: 999292928 | elapsed time per iteration (s): 5.39 | learning rate: 9.357E-04 | global batch size: 512 | lm loss: 3.900644E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.056 | TFLOPs: 71.80 | +[default7]: iteration 954/ 3814 | consumed samples: 488448 | consumed tokens: 1000341504 | elapsed time per iteration (s): 5.40 | learning rate: 9.355E-04 | global batch size: 512 | lm loss: 3.904723E+00 | loss scale: 8192.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.799 | TFLOPs: 71.60 | +[default7]: iteration 955/ 3814 | consumed samples: 488960 | consumed tokens: 1001390080 | elapsed time per iteration (s): 5.41 | learning rate: 9.353E-04 | global batch size: 512 | lm loss: 3.884491E+00 | loss scale: 8192.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.642 | TFLOPs: 71.48 | +[default7]: iteration 956/ 3814 | consumed samples: 489472 | consumed tokens: 1002438656 | elapsed time per iteration (s): 5.41 | learning rate: 9.351E-04 | global batch size: 512 | lm loss: 3.890490E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.717 | TFLOPs: 71.54 | +[default7]: iteration 957/ 3814 | consumed samples: 489984 | consumed tokens: 1003487232 | elapsed time per iteration (s): 5.41 | learning rate: 9.349E-04 | global batch size: 512 | lm loss: 3.888426E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.573 | TFLOPs: 71.43 | +[default7]: iteration 958/ 3814 | consumed samples: 490496 | consumed tokens: 1004535808 | elapsed time per iteration (s): 5.40 | learning rate: 9.347E-04 | global batch size: 512 | lm loss: 3.896480E+00 | loss scale: 8192.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.845 | TFLOPs: 71.64 | +[default7]: iteration 959/ 3814 | consumed samples: 491008 | consumed tokens: 1005584384 | elapsed time per iteration (s): 5.41 | learning rate: 9.345E-04 | global batch size: 512 | lm loss: 3.902503E+00 | loss scale: 8192.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.725 | TFLOPs: 71.55 | +[default7]: iteration 960/ 3814 | consumed samples: 491520 | consumed tokens: 1006632960 | elapsed time per iteration (s): 5.41 | learning rate: 9.342E-04 | global batch size: 512 | lm loss: 3.901017E+00 | loss scale: 8192.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.617 | TFLOPs: 71.47 | +[default7]: iteration 961/ 3814 | consumed samples: 492032 | consumed tokens: 1007681536 | elapsed time per iteration (s): 5.40 | learning rate: 9.340E-04 | global batch size: 512 | lm loss: 3.883205E+00 | loss scale: 8192.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.820 | TFLOPs: 71.62 | +[default7]: iteration 962/ 3814 | consumed samples: 492544 | consumed tokens: 1008730112 | elapsed time per iteration (s): 5.40 | learning rate: 9.338E-04 | global batch size: 512 | lm loss: 3.908447E+00 | loss scale: 8192.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.840 | TFLOPs: 71.63 | +[default7]: iteration 963/ 3814 | consumed samples: 493056 | consumed tokens: 1009778688 | elapsed time per iteration (s): 5.41 | learning rate: 9.336E-04 | global batch size: 512 | lm loss: 3.895823E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.681 | TFLOPs: 71.51 | +[default7]: iteration 964/ 3814 | consumed samples: 493568 | consumed tokens: 1010827264 | elapsed time per iteration (s): 5.41 | learning rate: 9.334E-04 | global batch size: 512 | lm loss: 3.912875E+00 | loss scale: 8192.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.564 | TFLOPs: 71.43 | +[default7]: iteration 965/ 3814 | consumed samples: 494080 | consumed tokens: 1011875840 | elapsed time per iteration (s): 5.41 | learning rate: 9.332E-04 | global batch size: 512 | lm loss: 3.893590E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.569 | TFLOPs: 71.43 | +[default7]: iteration 966/ 3814 | consumed samples: 494592 | consumed tokens: 1012924416 | elapsed time per iteration (s): 5.41 | learning rate: 9.330E-04 | global batch size: 512 | lm loss: 3.893389E+00 | loss scale: 8192.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.715 | TFLOPs: 71.54 | +[default7]: iteration 967/ 3814 | consumed samples: 495104 | consumed tokens: 1013972992 | elapsed time per iteration (s): 5.43 | learning rate: 9.327E-04 | global batch size: 512 | lm loss: 3.869524E+00 | loss scale: 8192.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 968/ 3814 | consumed samples: 495616 | consumed tokens: 1015021568 | elapsed time per iteration (s): 5.41 | learning rate: 9.325E-04 | global batch size: 512 | lm loss: 3.861217E+00 | loss scale: 8192.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.591 | TFLOPs: 71.45 | +[default7]: iteration 969/ 3814 | consumed samples: 496128 | consumed tokens: 1016070144 | elapsed time per iteration (s): 5.41 | learning rate: 9.323E-04 | global batch size: 512 | lm loss: 3.859733E+00 | loss scale: 8192.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 970/ 3814 | consumed samples: 496640 | consumed tokens: 1017118720 | elapsed time per iteration (s): 5.42 | learning rate: 9.321E-04 | global batch size: 512 | lm loss: 3.874331E+00 | loss scale: 8192.0 | grad norm: 0.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.479 | TFLOPs: 71.36 | +[default7]: iteration 971/ 3814 | consumed samples: 497152 | consumed tokens: 1018167296 | elapsed time per iteration (s): 5.40 | learning rate: 9.319E-04 | global batch size: 512 | lm loss: 3.875132E+00 | loss scale: 8192.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.735 | TFLOPs: 71.55 | +[default7]: iteration 972/ 3814 | consumed samples: 497664 | consumed tokens: 1019215872 | elapsed time per iteration (s): 5.43 | learning rate: 9.317E-04 | global batch size: 512 | lm loss: 3.881424E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.212 | TFLOPs: 71.16 | +[default7]: iteration 973/ 3814 | consumed samples: 498176 | consumed tokens: 1020264448 | elapsed time per iteration (s): 5.41 | learning rate: 9.315E-04 | global batch size: 512 | lm loss: 3.859501E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 974/ 3814 | consumed samples: 498688 | consumed tokens: 1021313024 | elapsed time per iteration (s): 5.41 | learning rate: 9.312E-04 | global batch size: 512 | lm loss: 3.879507E+00 | loss scale: 8192.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.624 | TFLOPs: 71.47 | +[default7]: iteration 975/ 3814 | consumed samples: 499200 | consumed tokens: 1022361600 | elapsed time per iteration (s): 5.39 | learning rate: 9.310E-04 | global batch size: 512 | lm loss: 3.884373E+00 | loss scale: 8192.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.931 | TFLOPs: 71.70 | +[default7]: iteration 976/ 3814 | consumed samples: 499712 | consumed tokens: 1023410176 | elapsed time per iteration (s): 5.39 | learning rate: 9.308E-04 | global batch size: 512 | lm loss: 3.907546E+00 | loss scale: 8192.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.909 | TFLOPs: 71.69 | +[default7]: iteration 977/ 3814 | consumed samples: 500224 | consumed tokens: 1024458752 | elapsed time per iteration (s): 5.41 | learning rate: 9.306E-04 | global batch size: 512 | lm loss: 3.893969E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.638 | TFLOPs: 71.48 | +[default7]: iteration 978/ 3814 | consumed samples: 500736 | consumed tokens: 1025507328 | elapsed time per iteration (s): 5.41 | learning rate: 9.304E-04 | global batch size: 512 | lm loss: 3.886460E+00 | loss scale: 8192.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.644 | TFLOPs: 71.49 | +[default7]: iteration 979/ 3814 | consumed samples: 501248 | consumed tokens: 1026555904 | elapsed time per iteration (s): 5.41 | learning rate: 9.301E-04 | global batch size: 512 | lm loss: 3.883282E+00 | loss scale: 8192.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.620 | TFLOPs: 71.47 | +[default7]: iteration 980/ 3814 | consumed samples: 501760 | consumed tokens: 1027604480 | elapsed time per iteration (s): 5.41 | learning rate: 9.299E-04 | global batch size: 512 | lm loss: 3.877764E+00 | loss scale: 8192.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.560 | TFLOPs: 71.42 | +[default7]: iteration 981/ 3814 | consumed samples: 502272 | consumed tokens: 1028653056 | elapsed time per iteration (s): 5.41 | learning rate: 9.297E-04 | global batch size: 512 | lm loss: 3.878140E+00 | loss scale: 8192.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.663 | TFLOPs: 71.50 | +[default7]: iteration 982/ 3814 | consumed samples: 502784 | consumed tokens: 1029701632 | elapsed time per iteration (s): 5.43 | learning rate: 9.295E-04 | global batch size: 512 | lm loss: 3.893286E+00 | loss scale: 8192.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.351 | TFLOPs: 71.26 | +[default7]: iteration 983/ 3814 | consumed samples: 503296 | consumed tokens: 1030750208 | elapsed time per iteration (s): 5.40 | learning rate: 9.293E-04 | global batch size: 512 | lm loss: 3.911283E+00 | loss scale: 8192.0 | grad norm: 0.140 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.775 | TFLOPs: 71.58 | +[default7]: iteration 984/ 3814 | consumed samples: 503808 | consumed tokens: 1031798784 | elapsed time per iteration (s): 5.41 | learning rate: 9.290E-04 | global batch size: 512 | lm loss: 3.877592E+00 | loss scale: 8192.0 | grad norm: 0.145 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.612 | TFLOPs: 71.46 | +[default7]: iteration 985/ 3814 | consumed samples: 504320 | consumed tokens: 1032847360 | elapsed time per iteration (s): 5.42 | learning rate: 9.288E-04 | global batch size: 512 | lm loss: 3.926757E+00 | loss scale: 8192.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default7]: iteration 986/ 3814 | consumed samples: 504832 | consumed tokens: 1033895936 | elapsed time per iteration (s): 5.42 | learning rate: 9.286E-04 | global batch size: 512 | lm loss: 3.876245E+00 | loss scale: 8192.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.512 | TFLOPs: 71.39 | +[default7]: iteration 987/ 3814 | consumed samples: 505344 | consumed tokens: 1034944512 | elapsed time per iteration (s): 5.41 | learning rate: 9.284E-04 | global batch size: 512 | lm loss: 3.886657E+00 | loss scale: 8192.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.727 | TFLOPs: 71.55 | +[default7]: iteration 988/ 3814 | consumed samples: 505856 | consumed tokens: 1035993088 | elapsed time per iteration (s): 5.40 | learning rate: 9.282E-04 | global batch size: 512 | lm loss: 3.883872E+00 | loss scale: 8192.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.764 | TFLOPs: 71.58 | +[default7]: iteration 989/ 3814 | consumed samples: 506368 | consumed tokens: 1037041664 | elapsed time per iteration (s): 5.40 | learning rate: 9.279E-04 | global batch size: 512 | lm loss: 3.887079E+00 | loss scale: 8192.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.753 | TFLOPs: 71.57 | +[default7]: iteration 990/ 3814 | consumed samples: 506880 | consumed tokens: 1038090240 | elapsed time per iteration (s): 5.42 | learning rate: 9.277E-04 | global batch size: 512 | lm loss: 3.905519E+00 | loss scale: 8192.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 991/ 3814 | consumed samples: 507392 | consumed tokens: 1039138816 | elapsed time per iteration (s): 5.40 | learning rate: 9.275E-04 | global batch size: 512 | lm loss: 3.857050E+00 | loss scale: 8192.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.867 | TFLOPs: 71.65 | +[default7]: iteration 992/ 3814 | consumed samples: 507904 | consumed tokens: 1040187392 | elapsed time per iteration (s): 5.42 | learning rate: 9.273E-04 | global batch size: 512 | lm loss: 3.832052E+00 | loss scale: 8192.0 | grad norm: 0.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.505 | TFLOPs: 71.38 | +[default7]: iteration 993/ 3814 | consumed samples: 508416 | consumed tokens: 1041235968 | elapsed time per iteration (s): 5.42 | learning rate: 9.271E-04 | global batch size: 512 | lm loss: 3.845355E+00 | loss scale: 8192.0 | grad norm: 0.073 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 994/ 3814 | consumed samples: 508928 | consumed tokens: 1042284544 | elapsed time per iteration (s): 5.41 | learning rate: 9.268E-04 | global batch size: 512 | lm loss: 3.866351E+00 | loss scale: 8192.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.595 | TFLOPs: 71.45 | +[default7]: iteration 995/ 3814 | consumed samples: 509440 | consumed tokens: 1043333120 | elapsed time per iteration (s): 5.42 | learning rate: 9.266E-04 | global batch size: 512 | lm loss: 3.855953E+00 | loss scale: 8192.0 | grad norm: 0.073 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.486 | TFLOPs: 71.37 | +[default7]: iteration 996/ 3814 | consumed samples: 509952 | consumed tokens: 1044381696 | elapsed time per iteration (s): 5.43 | learning rate: 9.264E-04 | global batch size: 512 | lm loss: 3.801271E+00 | loss scale: 8192.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.354 | TFLOPs: 71.27 | +[default7]: iteration 997/ 3814 | consumed samples: 510464 | consumed tokens: 1045430272 | elapsed time per iteration (s): 5.40 | learning rate: 9.262E-04 | global batch size: 512 | lm loss: 3.833266E+00 | loss scale: 8192.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.876 | TFLOPs: 71.66 | +[default7]: iteration 998/ 3814 | consumed samples: 510976 | consumed tokens: 1046478848 | elapsed time per iteration (s): 5.39 | learning rate: 9.259E-04 | global batch size: 512 | lm loss: 3.880934E+00 | loss scale: 8192.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.008 | TFLOPs: 71.76 | +[default7]: iteration 999/ 3814 | consumed samples: 511488 | consumed tokens: 1047527424 | elapsed time per iteration (s): 5.40 | learning rate: 9.257E-04 | global batch size: 512 | lm loss: 3.842036E+00 | loss scale: 8192.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.758 | TFLOPs: 71.57 | +[default7]: iteration 1000/ 3814 | consumed samples: 512000 | consumed tokens: 1048576000 | elapsed time per iteration (s): 5.41 | learning rate: 9.255E-04 | global batch size: 512 | lm loss: 3.857720E+00 | loss scale: 16384.0 | grad norm: 0.048 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.643 | TFLOPs: 71.49 | +[default0]:saving checkpoint at iteration 1000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 07:50:01,540] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is begin to save! +[default0]:[2023-02-16 07:50:01,546] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_01-model_00-model_states.pt... +[default7]:------------------------------------------------------------------------------------------ +[default7]:valid loss at iteration 1000 | lm loss value: 3.821792E+00 | lm loss PPL: 4.568602E+01 | +[default7]:------------------------------------------------------------------------------------------ +[default1]:[2023-02-16 07:50:01,546] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_01-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:01,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:01,718] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:01,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:01,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:01,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:01,858] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:01,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:01,815] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_05-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:01,874] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:01,875] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:01,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:01,933] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:01,976] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:01,976] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,039] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,025] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,103] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,146] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,147] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_10-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,165] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,166] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_09-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,208] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_11-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,268] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,269] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,287] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_11-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,348] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,327] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,328] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_13-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,390] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,391] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,419] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,420] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_13-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,453] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,453] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,483] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_14-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,550] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,551] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,514] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_16-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,581] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,581] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,613] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,614] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,643] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,644] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,677] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_17-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,741] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,706] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,771] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,805] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,836] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,837] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:02,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,898] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:02,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,869] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_20-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:02,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:02,944] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:02,976] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:02,976] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,032] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,043] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,103] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,103] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,112] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,113] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_25-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:03,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,170] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,170] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_24-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:03,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,315] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,315] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,315] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,408] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,407] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,408] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:03,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,487] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,487] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,488] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,560] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,568] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,569] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_29-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:03,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,632] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_30-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,620] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:03,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,679] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,700] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,741] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 07:50:03,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,777] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:03,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,838] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 07:50:03,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,811] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:03,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,870] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 07:50:03,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 07:50:03,873] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_01_model_states.pt +[default1]:[2023-02-16 07:50:03,873] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 07:50:03,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_01_model_states.pt. +[default0]:[2023-02-16 07:50:03,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,899] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_34-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:03,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:03,964] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 07:50:04,023] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:04,023] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_37-model_00-model_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:50:04,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 07:50:04,025] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_00_model_states.pt +[default0]:[2023-02-16 07:50:04,025] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 07:50:04,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/mp_rank_00_model_states.pt. +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 07:50:04,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 07:50:04,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,586] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,586] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,632] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,632] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,632] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,639] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,639] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,653] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,653] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,615] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,615] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,700] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,700] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,706] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,706] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,643] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,644] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,644] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,727] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,727] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,699] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,699] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,750] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,750] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,738] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,738] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,728] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,729] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default7]:[2023-02-16 07:50:04,704] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,704] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,704] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,678] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,678] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,678] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,675] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,675] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,747] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,747] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,768] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,768] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,771] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,771] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,795] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,795] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,795] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,751] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,751] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,763] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,763] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,794] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,794] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,794] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,795] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,774] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,774] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,785] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,785] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,785] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,823] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,823] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,780] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,780] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,793] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,793] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,757] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,757] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,796] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,796] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,796] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,800] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,800] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,772] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,772] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,816] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,816] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,842] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,842] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,789] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,789] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,789] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,823] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,824] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,803] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,803] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,803] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,805] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,806] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,767] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,767] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,809] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,809] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,766] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,767] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,781] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,781] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,781] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default1]:[2023-02-16 07:50:04,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 07:50:04,835] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 07:50:04,835] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,778] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,778] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,778] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:[2023-02-16 07:50:04,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:04,858] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:04,858] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,867] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,867] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,823] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,823] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:04,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:04,868] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:04,869] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,863] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,863] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,802] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,802] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:04,896] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:04,896] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:04,896] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]:[2023-02-16 07:50:04,832] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:04,832] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:04,832] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:04,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:04,857] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:04,857] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default3]:[2023-02-16 07:50:04,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 07:50:04,875] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 07:50:04,875] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default4]:[2023-02-16 07:50:04,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 07:50:04,903] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 07:50:04,903] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default6]:[2023-02-16 07:50:05,044] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 07:50:05,044] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 07:50:05,044] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default5]:[2023-02-16 07:50:05,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 07:50:05,040] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 07:50:05,040] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default2]:[2023-02-16 07:50:05,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 07:50:05,073] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 07:50:05,073] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default0]: successfully saved checkpoint at iteration 1000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]:[2023-02-16 07:50:05,113] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 07:50:05,114] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 07:50:05,114] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]:time (ms) | save-checkpoint: 3589.73 +[default0]:[2023-02-16 07:50:05,105] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 07:50:05,105] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1000/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 07:50:05,105] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! +[default7]: iteration 1001/ 3814 | consumed samples: 512512 | consumed tokens: 1049624576 | elapsed time per iteration (s): 10.02 | learning rate: 9.253E-04 | global batch size: 512 | lm loss: 3.871406E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 51.100 | TFLOPs: 38.60 | +[default7]: iteration 1002/ 3814 | consumed samples: 513024 | consumed tokens: 1050673152 | elapsed time per iteration (s): 5.40 | learning rate: 9.250E-04 | global batch size: 512 | lm loss: 3.877841E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.740 | TFLOPs: 71.56 | +[default7]: iteration 1003/ 3814 | consumed samples: 513536 | consumed tokens: 1051721728 | elapsed time per iteration (s): 5.42 | learning rate: 9.248E-04 | global batch size: 512 | lm loss: 3.831780E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 1004/ 3814 | consumed samples: 514048 | consumed tokens: 1052770304 | elapsed time per iteration (s): 5.41 | learning rate: 9.246E-04 | global batch size: 512 | lm loss: 3.844943E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.579 | TFLOPs: 71.44 | +[default7]: iteration 1005/ 3814 | consumed samples: 514560 | consumed tokens: 1053818880 | elapsed time per iteration (s): 5.43 | learning rate: 9.243E-04 | global batch size: 512 | lm loss: 3.835830E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.336 | TFLOPs: 71.25 | +[default7]: iteration 1006/ 3814 | consumed samples: 515072 | consumed tokens: 1054867456 | elapsed time per iteration (s): 5.42 | learning rate: 9.241E-04 | global batch size: 512 | lm loss: 3.876472E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 1007/ 3814 | consumed samples: 515584 | consumed tokens: 1055916032 | elapsed time per iteration (s): 5.42 | learning rate: 9.239E-04 | global batch size: 512 | lm loss: 3.838582E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 1008/ 3814 | consumed samples: 516096 | consumed tokens: 1056964608 | elapsed time per iteration (s): 5.42 | learning rate: 9.237E-04 | global batch size: 512 | lm loss: 3.829296E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.483 | TFLOPs: 71.36 | +[default7]: iteration 1009/ 3814 | consumed samples: 516608 | consumed tokens: 1058013184 | elapsed time per iteration (s): 5.44 | learning rate: 9.234E-04 | global batch size: 512 | lm loss: 3.841524E+00 | loss scale: 16384.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.191 | TFLOPs: 71.14 | +[default7]: iteration 1010/ 3814 | consumed samples: 517120 | consumed tokens: 1059061760 | elapsed time per iteration (s): 5.42 | learning rate: 9.232E-04 | global batch size: 512 | lm loss: 3.869354E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.427 | TFLOPs: 71.32 | +[default7]: iteration 1011/ 3814 | consumed samples: 517632 | consumed tokens: 1060110336 | elapsed time per iteration (s): 5.42 | learning rate: 9.230E-04 | global batch size: 512 | lm loss: 3.837540E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 1012/ 3814 | consumed samples: 518144 | consumed tokens: 1061158912 | elapsed time per iteration (s): 5.40 | learning rate: 9.228E-04 | global batch size: 512 | lm loss: 3.809551E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.873 | TFLOPs: 71.66 | +[default7]: iteration 1013/ 3814 | consumed samples: 518656 | consumed tokens: 1062207488 | elapsed time per iteration (s): 5.41 | learning rate: 9.225E-04 | global batch size: 512 | lm loss: 3.817486E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.673 | TFLOPs: 71.51 | +[default7]: iteration 1014/ 3814 | consumed samples: 519168 | consumed tokens: 1063256064 | elapsed time per iteration (s): 5.43 | learning rate: 9.223E-04 | global batch size: 512 | lm loss: 3.829354E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.336 | TFLOPs: 71.25 | +[default7]: iteration 1015/ 3814 | consumed samples: 519680 | consumed tokens: 1064304640 | elapsed time per iteration (s): 5.43 | learning rate: 9.221E-04 | global batch size: 512 | lm loss: 3.836024E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.354 | TFLOPs: 71.27 | +[default7]: iteration 1016/ 3814 | consumed samples: 520192 | consumed tokens: 1065353216 | elapsed time per iteration (s): 5.41 | learning rate: 9.218E-04 | global batch size: 512 | lm loss: 3.834514E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.590 | TFLOPs: 71.44 | +[default7]: iteration 1017/ 3814 | consumed samples: 520704 | consumed tokens: 1066401792 | elapsed time per iteration (s): 5.41 | learning rate: 9.216E-04 | global batch size: 512 | lm loss: 3.828188E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.686 | TFLOPs: 71.52 | +[default7]: iteration 1018/ 3814 | consumed samples: 521216 | consumed tokens: 1067450368 | elapsed time per iteration (s): 5.41 | learning rate: 9.214E-04 | global batch size: 512 | lm loss: 3.817291E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.554 | TFLOPs: 71.42 | +[default7]: iteration 1019/ 3814 | consumed samples: 521728 | consumed tokens: 1068498944 | elapsed time per iteration (s): 5.41 | learning rate: 9.211E-04 | global batch size: 512 | lm loss: 3.832105E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.691 | TFLOPs: 71.52 | +[default7]: iteration 1020/ 3814 | consumed samples: 522240 | consumed tokens: 1069547520 | elapsed time per iteration (s): 5.42 | learning rate: 9.209E-04 | global batch size: 512 | lm loss: 3.835266E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.466 | TFLOPs: 71.35 | +[default7]: iteration 1021/ 3814 | consumed samples: 522752 | consumed tokens: 1070596096 | elapsed time per iteration (s): 5.41 | learning rate: 9.207E-04 | global batch size: 512 | lm loss: 3.846037E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1022/ 3814 | consumed samples: 523264 | consumed tokens: 1071644672 | elapsed time per iteration (s): 5.40 | learning rate: 9.204E-04 | global batch size: 512 | lm loss: 3.827943E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.883 | TFLOPs: 71.67 | +[default7]: iteration 1023/ 3814 | consumed samples: 523776 | consumed tokens: 1072693248 | elapsed time per iteration (s): 5.43 | learning rate: 9.202E-04 | global batch size: 512 | lm loss: 3.799833E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.369 | TFLOPs: 71.28 | +[default7]: iteration 1024/ 3814 | consumed samples: 524288 | consumed tokens: 1073741824 | elapsed time per iteration (s): 5.43 | learning rate: 9.200E-04 | global batch size: 512 | lm loss: 3.806231E+00 | loss scale: 16384.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.264 | TFLOPs: 71.20 | +[default7]: iteration 1025/ 3814 | consumed samples: 524800 | consumed tokens: 1074790400 | elapsed time per iteration (s): 5.42 | learning rate: 9.197E-04 | global batch size: 512 | lm loss: 3.820635E+00 | loss scale: 16384.0 | grad norm: 0.066 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 1026/ 3814 | consumed samples: 525312 | consumed tokens: 1075838976 | elapsed time per iteration (s): 5.41 | learning rate: 9.195E-04 | global batch size: 512 | lm loss: 3.793929E+00 | loss scale: 16384.0 | grad norm: 0.062 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.641 | TFLOPs: 71.48 | +[default7]: iteration 1027/ 3814 | consumed samples: 525824 | consumed tokens: 1076887552 | elapsed time per iteration (s): 5.42 | learning rate: 9.193E-04 | global batch size: 512 | lm loss: 3.801590E+00 | loss scale: 16384.0 | grad norm: 0.063 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.463 | TFLOPs: 71.35 | +[default7]: iteration 1028/ 3814 | consumed samples: 526336 | consumed tokens: 1077936128 | elapsed time per iteration (s): 5.42 | learning rate: 9.190E-04 | global batch size: 512 | lm loss: 3.814970E+00 | loss scale: 16384.0 | grad norm: 0.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.529 | TFLOPs: 71.40 | +[default7]: iteration 1029/ 3814 | consumed samples: 526848 | consumed tokens: 1078984704 | elapsed time per iteration (s): 5.42 | learning rate: 9.188E-04 | global batch size: 512 | lm loss: 3.808684E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.457 | TFLOPs: 71.34 | +[default7]: iteration 1030/ 3814 | consumed samples: 527360 | consumed tokens: 1080033280 | elapsed time per iteration (s): 5.42 | learning rate: 9.186E-04 | global batch size: 512 | lm loss: 3.776988E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.454 | TFLOPs: 71.34 | +[default7]: iteration 1031/ 3814 | consumed samples: 527872 | consumed tokens: 1081081856 | elapsed time per iteration (s): 5.43 | learning rate: 9.183E-04 | global batch size: 512 | lm loss: 3.820508E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.352 | TFLOPs: 71.27 | +[default7]: iteration 1032/ 3814 | consumed samples: 528384 | consumed tokens: 1082130432 | elapsed time per iteration (s): 5.47 | learning rate: 9.181E-04 | global batch size: 512 | lm loss: 3.798527E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.669 | TFLOPs: 70.75 | +[default7]: iteration 1033/ 3814 | consumed samples: 528896 | consumed tokens: 1083179008 | elapsed time per iteration (s): 5.42 | learning rate: 9.179E-04 | global batch size: 512 | lm loss: 3.795813E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 1034/ 3814 | consumed samples: 529408 | consumed tokens: 1084227584 | elapsed time per iteration (s): 5.41 | learning rate: 9.176E-04 | global batch size: 512 | lm loss: 3.850655E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.579 | TFLOPs: 71.44 | +[default7]: iteration 1035/ 3814 | consumed samples: 529920 | consumed tokens: 1085276160 | elapsed time per iteration (s): 5.41 | learning rate: 9.174E-04 | global batch size: 512 | lm loss: 3.812226E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.678 | TFLOPs: 71.51 | +[default7]: iteration 1036/ 3814 | consumed samples: 530432 | consumed tokens: 1086324736 | elapsed time per iteration (s): 5.42 | learning rate: 9.172E-04 | global batch size: 512 | lm loss: 3.796237E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.408 | TFLOPs: 71.31 | +[default7]: iteration 1037/ 3814 | consumed samples: 530944 | consumed tokens: 1087373312 | elapsed time per iteration (s): 5.43 | learning rate: 9.169E-04 | global batch size: 512 | lm loss: 3.773797E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.218 | TFLOPs: 71.16 | +[default7]: iteration 1038/ 3814 | consumed samples: 531456 | consumed tokens: 1088421888 | elapsed time per iteration (s): 5.43 | learning rate: 9.167E-04 | global batch size: 512 | lm loss: 3.833760E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 1039/ 3814 | consumed samples: 531968 | consumed tokens: 1089470464 | elapsed time per iteration (s): 5.41 | learning rate: 9.165E-04 | global batch size: 512 | lm loss: 3.802846E+00 | loss scale: 16384.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.584 | TFLOPs: 71.44 | +[default7]: iteration 1040/ 3814 | consumed samples: 532480 | consumed tokens: 1090519040 | elapsed time per iteration (s): 5.41 | learning rate: 9.162E-04 | global batch size: 512 | lm loss: 3.809823E+00 | loss scale: 16384.0 | grad norm: 0.142 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 1041/ 3814 | consumed samples: 532992 | consumed tokens: 1091567616 | elapsed time per iteration (s): 5.41 | learning rate: 9.160E-04 | global batch size: 512 | lm loss: 3.791380E+00 | loss scale: 16384.0 | grad norm: 0.137 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 1042/ 3814 | consumed samples: 533504 | consumed tokens: 1092616192 | elapsed time per iteration (s): 5.42 | learning rate: 9.157E-04 | global batch size: 512 | lm loss: 3.820082E+00 | loss scale: 16384.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.408 | TFLOPs: 71.31 | +[default7]: iteration 1043/ 3814 | consumed samples: 534016 | consumed tokens: 1093664768 | elapsed time per iteration (s): 5.41 | learning rate: 9.155E-04 | global batch size: 512 | lm loss: 3.808003E+00 | loss scale: 16384.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.606 | TFLOPs: 71.46 | +[default7]: iteration 1044/ 3814 | consumed samples: 534528 | consumed tokens: 1094713344 | elapsed time per iteration (s): 5.41 | learning rate: 9.153E-04 | global batch size: 512 | lm loss: 3.799842E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.638 | TFLOPs: 71.48 | +[default7]: iteration 1045/ 3814 | consumed samples: 535040 | consumed tokens: 1095761920 | elapsed time per iteration (s): 5.41 | learning rate: 9.150E-04 | global batch size: 512 | lm loss: 3.789371E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.569 | TFLOPs: 71.43 | +[default7]: iteration 1046/ 3814 | consumed samples: 535552 | consumed tokens: 1096810496 | elapsed time per iteration (s): 5.41 | learning rate: 9.148E-04 | global batch size: 512 | lm loss: 3.766010E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.707 | TFLOPs: 71.53 | +[default7]: iteration 1047/ 3814 | consumed samples: 536064 | consumed tokens: 1097859072 | elapsed time per iteration (s): 5.40 | learning rate: 9.145E-04 | global batch size: 512 | lm loss: 3.783273E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.840 | TFLOPs: 71.63 | +[default7]: iteration 1048/ 3814 | consumed samples: 536576 | consumed tokens: 1098907648 | elapsed time per iteration (s): 5.41 | learning rate: 9.143E-04 | global batch size: 512 | lm loss: 3.823936E+00 | loss scale: 16384.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.649 | TFLOPs: 71.49 | +[default7]: iteration 1049/ 3814 | consumed samples: 537088 | consumed tokens: 1099956224 | elapsed time per iteration (s): 5.42 | learning rate: 9.141E-04 | global batch size: 512 | lm loss: 3.795834E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.530 | TFLOPs: 71.40 | +[default7]: iteration 1050/ 3814 | consumed samples: 537600 | consumed tokens: 1101004800 | elapsed time per iteration (s): 5.41 | learning rate: 9.138E-04 | global batch size: 512 | lm loss: 3.804685E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.589 | TFLOPs: 71.44 | +[default7]: iteration 1051/ 3814 | consumed samples: 538112 | consumed tokens: 1102053376 | elapsed time per iteration (s): 5.42 | learning rate: 9.136E-04 | global batch size: 512 | lm loss: 3.778045E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.392 | TFLOPs: 71.29 | +[default7]: iteration 1052/ 3814 | consumed samples: 538624 | consumed tokens: 1103101952 | elapsed time per iteration (s): 5.41 | learning rate: 9.133E-04 | global batch size: 512 | lm loss: 3.805200E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.679 | TFLOPs: 71.51 | +[default7]: iteration 1053/ 3814 | consumed samples: 539136 | consumed tokens: 1104150528 | elapsed time per iteration (s): 5.43 | learning rate: 9.131E-04 | global batch size: 512 | lm loss: 3.779612E+00 | loss scale: 16384.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.324 | TFLOPs: 71.24 | +[default7]: iteration 1054/ 3814 | consumed samples: 539648 | consumed tokens: 1105199104 | elapsed time per iteration (s): 5.43 | learning rate: 9.129E-04 | global batch size: 512 | lm loss: 3.780827E+00 | loss scale: 16384.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.266 | TFLOPs: 71.20 | +[default7]: iteration 1055/ 3814 | consumed samples: 540160 | consumed tokens: 1106247680 | elapsed time per iteration (s): 5.43 | learning rate: 9.126E-04 | global batch size: 512 | lm loss: 3.783248E+00 | loss scale: 16384.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1056/ 3814 | consumed samples: 540672 | consumed tokens: 1107296256 | elapsed time per iteration (s): 5.42 | learning rate: 9.124E-04 | global batch size: 512 | lm loss: 3.808112E+00 | loss scale: 16384.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 1057/ 3814 | consumed samples: 541184 | consumed tokens: 1108344832 | elapsed time per iteration (s): 5.42 | learning rate: 9.121E-04 | global batch size: 512 | lm loss: 3.843747E+00 | loss scale: 16384.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.498 | TFLOPs: 71.38 | +[default7]: iteration 1058/ 3814 | consumed samples: 541696 | consumed tokens: 1109393408 | elapsed time per iteration (s): 5.40 | learning rate: 9.119E-04 | global batch size: 512 | lm loss: 3.783156E+00 | loss scale: 16384.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 1059/ 3814 | consumed samples: 542208 | consumed tokens: 1110441984 | elapsed time per iteration (s): 5.42 | learning rate: 9.116E-04 | global batch size: 512 | lm loss: 3.788577E+00 | loss scale: 16384.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.508 | TFLOPs: 71.38 | +[default7]: iteration 1060/ 3814 | consumed samples: 542720 | consumed tokens: 1111490560 | elapsed time per iteration (s): 5.43 | learning rate: 9.114E-04 | global batch size: 512 | lm loss: 3.768583E+00 | loss scale: 16384.0 | grad norm: 0.073 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.362 | TFLOPs: 71.27 | +[default7]: iteration 1061/ 3814 | consumed samples: 543232 | consumed tokens: 1112539136 | elapsed time per iteration (s): 5.43 | learning rate: 9.112E-04 | global batch size: 512 | lm loss: 3.789253E+00 | loss scale: 16384.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.299 | TFLOPs: 71.22 | +[default7]: iteration 1062/ 3814 | consumed samples: 543744 | consumed tokens: 1113587712 | elapsed time per iteration (s): 5.43 | learning rate: 9.109E-04 | global batch size: 512 | lm loss: 3.725899E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.341 | TFLOPs: 71.26 | +[default7]: iteration 1063/ 3814 | consumed samples: 544256 | consumed tokens: 1114636288 | elapsed time per iteration (s): 5.43 | learning rate: 9.107E-04 | global batch size: 512 | lm loss: 3.752499E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.209 | TFLOPs: 71.16 | +[default7]: iteration 1064/ 3814 | consumed samples: 544768 | consumed tokens: 1115684864 | elapsed time per iteration (s): 5.42 | learning rate: 9.104E-04 | global batch size: 512 | lm loss: 3.778573E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1065/ 3814 | consumed samples: 545280 | consumed tokens: 1116733440 | elapsed time per iteration (s): 5.44 | learning rate: 9.102E-04 | global batch size: 512 | lm loss: 3.790719E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.150 | TFLOPs: 71.11 | +[default7]: iteration 1066/ 3814 | consumed samples: 545792 | consumed tokens: 1117782016 | elapsed time per iteration (s): 5.43 | learning rate: 9.099E-04 | global batch size: 512 | lm loss: 3.787259E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.243 | TFLOPs: 71.18 | +[default7]: iteration 1067/ 3814 | consumed samples: 546304 | consumed tokens: 1118830592 | elapsed time per iteration (s): 5.40 | learning rate: 9.097E-04 | global batch size: 512 | lm loss: 3.747467E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.832 | TFLOPs: 71.63 | +[default7]: iteration 1068/ 3814 | consumed samples: 546816 | consumed tokens: 1119879168 | elapsed time per iteration (s): 5.40 | learning rate: 9.094E-04 | global batch size: 512 | lm loss: 3.762001E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.812 | TFLOPs: 71.61 | +[default7]: iteration 1069/ 3814 | consumed samples: 547328 | consumed tokens: 1120927744 | elapsed time per iteration (s): 5.41 | learning rate: 9.092E-04 | global batch size: 512 | lm loss: 3.761409E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.662 | TFLOPs: 71.50 | +[default7]: iteration 1070/ 3814 | consumed samples: 547840 | consumed tokens: 1121976320 | elapsed time per iteration (s): 5.41 | learning rate: 9.090E-04 | global batch size: 512 | lm loss: 3.778174E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.704 | TFLOPs: 71.53 | +[default7]: iteration 1071/ 3814 | consumed samples: 548352 | consumed tokens: 1123024896 | elapsed time per iteration (s): 5.43 | learning rate: 9.087E-04 | global batch size: 512 | lm loss: 3.786737E+00 | loss scale: 16384.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.291 | TFLOPs: 71.22 | +[default7]: iteration 1072/ 3814 | consumed samples: 548864 | consumed tokens: 1124073472 | elapsed time per iteration (s): 5.44 | learning rate: 9.085E-04 | global batch size: 512 | lm loss: 3.757806E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.097 | TFLOPs: 71.07 | +[default7]: iteration 1073/ 3814 | consumed samples: 549376 | consumed tokens: 1125122048 | elapsed time per iteration (s): 5.42 | learning rate: 9.082E-04 | global batch size: 512 | lm loss: 3.769897E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.396 | TFLOPs: 71.30 | +[default7]: iteration 1074/ 3814 | consumed samples: 549888 | consumed tokens: 1126170624 | elapsed time per iteration (s): 5.41 | learning rate: 9.080E-04 | global batch size: 512 | lm loss: 3.764891E+00 | loss scale: 16384.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.645 | TFLOPs: 71.49 | +[default7]: iteration 1075/ 3814 | consumed samples: 550400 | consumed tokens: 1127219200 | elapsed time per iteration (s): 5.42 | learning rate: 9.077E-04 | global batch size: 512 | lm loss: 3.775611E+00 | loss scale: 16384.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 1076/ 3814 | consumed samples: 550912 | consumed tokens: 1128267776 | elapsed time per iteration (s): 5.42 | learning rate: 9.075E-04 | global batch size: 512 | lm loss: 3.764844E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.521 | TFLOPs: 71.39 | +[default7]: iteration 1077/ 3814 | consumed samples: 551424 | consumed tokens: 1129316352 | elapsed time per iteration (s): 5.43 | learning rate: 9.072E-04 | global batch size: 512 | lm loss: 3.752570E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.332 | TFLOPs: 71.25 | +[default7]: iteration 1078/ 3814 | consumed samples: 551936 | consumed tokens: 1130364928 | elapsed time per iteration (s): 5.40 | learning rate: 9.070E-04 | global batch size: 512 | lm loss: 3.779775E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.785 | TFLOPs: 71.59 | +[default7]: iteration 1079/ 3814 | consumed samples: 552448 | consumed tokens: 1131413504 | elapsed time per iteration (s): 5.38 | learning rate: 9.067E-04 | global batch size: 512 | lm loss: 3.745123E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.079 | TFLOPs: 71.81 | +[default7]: iteration 1080/ 3814 | consumed samples: 552960 | consumed tokens: 1132462080 | elapsed time per iteration (s): 5.42 | learning rate: 9.065E-04 | global batch size: 512 | lm loss: 3.761124E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.545 | TFLOPs: 71.41 | +[default7]: iteration 1081/ 3814 | consumed samples: 553472 | consumed tokens: 1133510656 | elapsed time per iteration (s): 5.43 | learning rate: 9.062E-04 | global batch size: 512 | lm loss: 3.771125E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.298 | TFLOPs: 71.22 | +[default7]: iteration 1082/ 3814 | consumed samples: 553984 | consumed tokens: 1134559232 | elapsed time per iteration (s): 5.42 | learning rate: 9.060E-04 | global batch size: 512 | lm loss: 3.772724E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.528 | TFLOPs: 71.40 | +[default7]: iteration 1083/ 3814 | consumed samples: 554496 | consumed tokens: 1135607808 | elapsed time per iteration (s): 5.43 | learning rate: 9.057E-04 | global batch size: 512 | lm loss: 3.776680E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.298 | TFLOPs: 71.22 | +[default7]: iteration 1084/ 3814 | consumed samples: 555008 | consumed tokens: 1136656384 | elapsed time per iteration (s): 5.41 | learning rate: 9.055E-04 | global batch size: 512 | lm loss: 3.754239E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.722 | TFLOPs: 71.54 | +[default7]: iteration 1085/ 3814 | consumed samples: 555520 | consumed tokens: 1137704960 | elapsed time per iteration (s): 5.40 | learning rate: 9.052E-04 | global batch size: 512 | lm loss: 3.775440E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.893 | TFLOPs: 71.67 | +[default7]: iteration 1086/ 3814 | consumed samples: 556032 | consumed tokens: 1138753536 | elapsed time per iteration (s): 5.41 | learning rate: 9.050E-04 | global batch size: 512 | lm loss: 3.740876E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.655 | TFLOPs: 71.49 | +[default7]: iteration 1087/ 3814 | consumed samples: 556544 | consumed tokens: 1139802112 | elapsed time per iteration (s): 5.42 | learning rate: 9.047E-04 | global batch size: 512 | lm loss: 3.739363E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.507 | TFLOPs: 71.38 | +[default7]: iteration 1088/ 3814 | consumed samples: 557056 | consumed tokens: 1140850688 | elapsed time per iteration (s): 5.42 | learning rate: 9.045E-04 | global batch size: 512 | lm loss: 3.747229E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.442 | TFLOPs: 71.33 | +[default7]: iteration 1089/ 3814 | consumed samples: 557568 | consumed tokens: 1141899264 | elapsed time per iteration (s): 5.41 | learning rate: 9.042E-04 | global batch size: 512 | lm loss: 3.770875E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 1090/ 3814 | consumed samples: 558080 | consumed tokens: 1142947840 | elapsed time per iteration (s): 5.43 | learning rate: 9.040E-04 | global batch size: 512 | lm loss: 3.764463E+00 | loss scale: 16384.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.221 | TFLOPs: 71.17 | +[default7]: iteration 1091/ 3814 | consumed samples: 558592 | consumed tokens: 1143996416 | elapsed time per iteration (s): 5.41 | learning rate: 9.037E-04 | global batch size: 512 | lm loss: 3.729173E+00 | loss scale: 16384.0 | grad norm: 0.070 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 1092/ 3814 | consumed samples: 559104 | consumed tokens: 1145044992 | elapsed time per iteration (s): 5.41 | learning rate: 9.035E-04 | global batch size: 512 | lm loss: 3.735397E+00 | loss scale: 16384.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 1093/ 3814 | consumed samples: 559616 | consumed tokens: 1146093568 | elapsed time per iteration (s): 5.43 | learning rate: 9.032E-04 | global batch size: 512 | lm loss: 3.739355E+00 | loss scale: 16384.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.363 | TFLOPs: 71.27 | +[default7]: iteration 1094/ 3814 | consumed samples: 560128 | consumed tokens: 1147142144 | elapsed time per iteration (s): 5.44 | learning rate: 9.030E-04 | global batch size: 512 | lm loss: 3.752743E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.092 | TFLOPs: 71.07 | +[default7]: iteration 1095/ 3814 | consumed samples: 560640 | consumed tokens: 1148190720 | elapsed time per iteration (s): 5.42 | learning rate: 9.027E-04 | global batch size: 512 | lm loss: 3.738248E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 1096/ 3814 | consumed samples: 561152 | consumed tokens: 1149239296 | elapsed time per iteration (s): 5.42 | learning rate: 9.024E-04 | global batch size: 512 | lm loss: 3.750658E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.468 | TFLOPs: 71.35 | +[default7]: iteration 1097/ 3814 | consumed samples: 561664 | consumed tokens: 1150287872 | elapsed time per iteration (s): 5.42 | learning rate: 9.022E-04 | global batch size: 512 | lm loss: 3.763352E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 1098/ 3814 | consumed samples: 562176 | consumed tokens: 1151336448 | elapsed time per iteration (s): 5.43 | learning rate: 9.019E-04 | global batch size: 512 | lm loss: 3.750788E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.332 | TFLOPs: 71.25 | +[default7]: iteration 1099/ 3814 | consumed samples: 562688 | consumed tokens: 1152385024 | elapsed time per iteration (s): 5.44 | learning rate: 9.017E-04 | global batch size: 512 | lm loss: 3.727999E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.057 | TFLOPs: 71.04 | +[default7]: iteration 1100/ 3814 | consumed samples: 563200 | consumed tokens: 1153433600 | elapsed time per iteration (s): 5.44 | learning rate: 9.014E-04 | global batch size: 512 | lm loss: 3.726556E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.111 | TFLOPs: 71.08 | +[default7]: iteration 1101/ 3814 | consumed samples: 563712 | consumed tokens: 1154482176 | elapsed time per iteration (s): 5.42 | learning rate: 9.012E-04 | global batch size: 512 | lm loss: 3.725987E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.449 | TFLOPs: 71.34 | +[default7]: iteration 1102/ 3814 | consumed samples: 564224 | consumed tokens: 1155530752 | elapsed time per iteration (s): 5.42 | learning rate: 9.009E-04 | global batch size: 512 | lm loss: 3.748164E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.389 | TFLOPs: 71.29 | +[default7]: iteration 1103/ 3814 | consumed samples: 564736 | consumed tokens: 1156579328 | elapsed time per iteration (s): 5.44 | learning rate: 9.007E-04 | global batch size: 512 | lm loss: 3.734347E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.108 | TFLOPs: 71.08 | +[default7]: iteration 1104/ 3814 | consumed samples: 565248 | consumed tokens: 1157627904 | elapsed time per iteration (s): 5.43 | learning rate: 9.004E-04 | global batch size: 512 | lm loss: 3.735965E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1105/ 3814 | consumed samples: 565760 | consumed tokens: 1158676480 | elapsed time per iteration (s): 5.44 | learning rate: 9.001E-04 | global batch size: 512 | lm loss: 3.759674E+00 | loss scale: 16384.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.174 | TFLOPs: 71.13 | +[default7]: iteration 1106/ 3814 | consumed samples: 566272 | consumed tokens: 1159725056 | elapsed time per iteration (s): 5.43 | learning rate: 8.999E-04 | global batch size: 512 | lm loss: 3.730107E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.274 | TFLOPs: 71.21 | +[default7]: iteration 1107/ 3814 | consumed samples: 566784 | consumed tokens: 1160773632 | elapsed time per iteration (s): 5.43 | learning rate: 8.996E-04 | global batch size: 512 | lm loss: 3.742512E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 1108/ 3814 | consumed samples: 567296 | consumed tokens: 1161822208 | elapsed time per iteration (s): 5.43 | learning rate: 8.994E-04 | global batch size: 512 | lm loss: 3.727683E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 1109/ 3814 | consumed samples: 567808 | consumed tokens: 1162870784 | elapsed time per iteration (s): 5.43 | learning rate: 8.991E-04 | global batch size: 512 | lm loss: 3.721874E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.253 | TFLOPs: 71.19 | +[default7]: iteration 1110/ 3814 | consumed samples: 568320 | consumed tokens: 1163919360 | elapsed time per iteration (s): 5.45 | learning rate: 8.989E-04 | global batch size: 512 | lm loss: 3.752623E+00 | loss scale: 16384.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.876 | TFLOPs: 70.91 | +[default7]: iteration 1111/ 3814 | consumed samples: 568832 | consumed tokens: 1164967936 | elapsed time per iteration (s): 5.43 | learning rate: 8.986E-04 | global batch size: 512 | lm loss: 3.740541E+00 | loss scale: 16384.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.226 | TFLOPs: 71.17 | +[default7]: iteration 1112/ 3814 | consumed samples: 569344 | consumed tokens: 1166016512 | elapsed time per iteration (s): 5.43 | learning rate: 8.983E-04 | global batch size: 512 | lm loss: 3.741332E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.279 | TFLOPs: 71.21 | +[default7]: iteration 1113/ 3814 | consumed samples: 569856 | consumed tokens: 1167065088 | elapsed time per iteration (s): 5.43 | learning rate: 8.981E-04 | global batch size: 512 | lm loss: 3.720878E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.266 | TFLOPs: 71.20 | +[default7]: iteration 1114/ 3814 | consumed samples: 570368 | consumed tokens: 1168113664 | elapsed time per iteration (s): 5.44 | learning rate: 8.978E-04 | global batch size: 512 | lm loss: 3.733139E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.076 | TFLOPs: 71.06 | +[default7]: iteration 1115/ 3814 | consumed samples: 570880 | consumed tokens: 1169162240 | elapsed time per iteration (s): 5.42 | learning rate: 8.976E-04 | global batch size: 512 | lm loss: 3.727370E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.547 | TFLOPs: 71.41 | +[default7]: iteration 1116/ 3814 | consumed samples: 571392 | consumed tokens: 1170210816 | elapsed time per iteration (s): 5.42 | learning rate: 8.973E-04 | global batch size: 512 | lm loss: 3.741405E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.433 | TFLOPs: 71.33 | +[default7]: iteration 1117/ 3814 | consumed samples: 571904 | consumed tokens: 1171259392 | elapsed time per iteration (s): 5.42 | learning rate: 8.970E-04 | global batch size: 512 | lm loss: 3.744419E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.391 | TFLOPs: 71.29 | +[default7]: iteration 1118/ 3814 | consumed samples: 572416 | consumed tokens: 1172307968 | elapsed time per iteration (s): 5.42 | learning rate: 8.968E-04 | global batch size: 512 | lm loss: 3.745188E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.510 | TFLOPs: 71.38 | +[default7]: iteration 1119/ 3814 | consumed samples: 572928 | consumed tokens: 1173356544 | elapsed time per iteration (s): 5.42 | learning rate: 8.965E-04 | global batch size: 512 | lm loss: 3.752955E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.482 | TFLOPs: 71.36 | +[default7]: iteration 1120/ 3814 | consumed samples: 573440 | consumed tokens: 1174405120 | elapsed time per iteration (s): 5.41 | learning rate: 8.963E-04 | global batch size: 512 | lm loss: 3.737113E+00 | loss scale: 16384.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.664 | TFLOPs: 71.50 | +[default7]: iteration 1121/ 3814 | consumed samples: 573952 | consumed tokens: 1175453696 | elapsed time per iteration (s): 5.41 | learning rate: 8.960E-04 | global batch size: 512 | lm loss: 3.724051E+00 | loss scale: 16384.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.573 | TFLOPs: 71.43 | +[default7]: iteration 1122/ 3814 | consumed samples: 574464 | consumed tokens: 1176502272 | elapsed time per iteration (s): 5.42 | learning rate: 8.957E-04 | global batch size: 512 | lm loss: 3.737894E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.430 | TFLOPs: 71.32 | +[default7]: iteration 1123/ 3814 | consumed samples: 574976 | consumed tokens: 1177550848 | elapsed time per iteration (s): 5.43 | learning rate: 8.955E-04 | global batch size: 512 | lm loss: 3.723119E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.272 | TFLOPs: 71.20 | +[default7]: iteration 1124/ 3814 | consumed samples: 575488 | consumed tokens: 1178599424 | elapsed time per iteration (s): 5.43 | learning rate: 8.952E-04 | global batch size: 512 | lm loss: 3.704699E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.250 | TFLOPs: 71.19 | +[default7]: iteration 1125/ 3814 | consumed samples: 576000 | consumed tokens: 1179648000 | elapsed time per iteration (s): 5.42 | learning rate: 8.949E-04 | global batch size: 512 | lm loss: 3.722679E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.415 | TFLOPs: 71.31 | +[default7]: iteration 1126/ 3814 | consumed samples: 576512 | consumed tokens: 1180696576 | elapsed time per iteration (s): 5.42 | learning rate: 8.947E-04 | global batch size: 512 | lm loss: 3.739037E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.403 | TFLOPs: 71.30 | +[default7]: iteration 1127/ 3814 | consumed samples: 577024 | consumed tokens: 1181745152 | elapsed time per iteration (s): 5.40 | learning rate: 8.944E-04 | global batch size: 512 | lm loss: 3.706913E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.740 | TFLOPs: 71.56 | +[default7]: iteration 1128/ 3814 | consumed samples: 577536 | consumed tokens: 1182793728 | elapsed time per iteration (s): 5.43 | learning rate: 8.942E-04 | global batch size: 512 | lm loss: 3.705338E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.215 | TFLOPs: 71.16 | +[default7]: iteration 1129/ 3814 | consumed samples: 578048 | consumed tokens: 1183842304 | elapsed time per iteration (s): 5.42 | learning rate: 8.939E-04 | global batch size: 512 | lm loss: 3.729803E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.515 | TFLOPs: 71.39 | +[default7]: iteration 1130/ 3814 | consumed samples: 578560 | consumed tokens: 1184890880 | elapsed time per iteration (s): 5.42 | learning rate: 8.936E-04 | global batch size: 512 | lm loss: 3.723781E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 1131/ 3814 | consumed samples: 579072 | consumed tokens: 1185939456 | elapsed time per iteration (s): 5.41 | learning rate: 8.934E-04 | global batch size: 512 | lm loss: 3.719707E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.688 | TFLOPs: 71.52 | +[default7]: iteration 1132/ 3814 | consumed samples: 579584 | consumed tokens: 1186988032 | elapsed time per iteration (s): 5.41 | learning rate: 8.931E-04 | global batch size: 512 | lm loss: 3.719402E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1133/ 3814 | consumed samples: 580096 | consumed tokens: 1188036608 | elapsed time per iteration (s): 5.41 | learning rate: 8.928E-04 | global batch size: 512 | lm loss: 3.733995E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.591 | TFLOPs: 71.45 | +[default7]: iteration 1134/ 3814 | consumed samples: 580608 | consumed tokens: 1189085184 | elapsed time per iteration (s): 5.43 | learning rate: 8.926E-04 | global batch size: 512 | lm loss: 3.715168E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.232 | TFLOPs: 71.17 | +[default7]: iteration 1135/ 3814 | consumed samples: 581120 | consumed tokens: 1190133760 | elapsed time per iteration (s): 5.42 | learning rate: 8.923E-04 | global batch size: 512 | lm loss: 3.676423E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 1136/ 3814 | consumed samples: 581632 | consumed tokens: 1191182336 | elapsed time per iteration (s): 5.42 | learning rate: 8.920E-04 | global batch size: 512 | lm loss: 3.671628E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.470 | TFLOPs: 71.35 | +[default7]: iteration 1137/ 3814 | consumed samples: 582144 | consumed tokens: 1192230912 | elapsed time per iteration (s): 5.43 | learning rate: 8.918E-04 | global batch size: 512 | lm loss: 3.700051E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.314 | TFLOPs: 71.24 | +[default7]: iteration 1138/ 3814 | consumed samples: 582656 | consumed tokens: 1193279488 | elapsed time per iteration (s): 5.42 | learning rate: 8.915E-04 | global batch size: 512 | lm loss: 3.712628E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.453 | TFLOPs: 71.34 | +[default7]: iteration 1139/ 3814 | consumed samples: 583168 | consumed tokens: 1194328064 | elapsed time per iteration (s): 5.43 | learning rate: 8.912E-04 | global batch size: 512 | lm loss: 3.697681E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.278 | TFLOPs: 71.21 | +[default7]: iteration 1140/ 3814 | consumed samples: 583680 | consumed tokens: 1195376640 | elapsed time per iteration (s): 5.42 | learning rate: 8.910E-04 | global batch size: 512 | lm loss: 3.688244E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.497 | TFLOPs: 71.37 | +[default7]: iteration 1141/ 3814 | consumed samples: 584192 | consumed tokens: 1196425216 | elapsed time per iteration (s): 5.42 | learning rate: 8.907E-04 | global batch size: 512 | lm loss: 3.698969E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.552 | TFLOPs: 71.42 | +[default7]: iteration 1142/ 3814 | consumed samples: 584704 | consumed tokens: 1197473792 | elapsed time per iteration (s): 5.41 | learning rate: 8.904E-04 | global batch size: 512 | lm loss: 3.690871E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 1143/ 3814 | consumed samples: 585216 | consumed tokens: 1198522368 | elapsed time per iteration (s): 5.41 | learning rate: 8.902E-04 | global batch size: 512 | lm loss: 3.690117E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 1144/ 3814 | consumed samples: 585728 | consumed tokens: 1199570944 | elapsed time per iteration (s): 5.40 | learning rate: 8.899E-04 | global batch size: 512 | lm loss: 3.685159E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.731 | TFLOPs: 71.55 | +[default7]: iteration 1145/ 3814 | consumed samples: 586240 | consumed tokens: 1200619520 | elapsed time per iteration (s): 5.43 | learning rate: 8.896E-04 | global batch size: 512 | lm loss: 3.689707E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.377 | TFLOPs: 71.28 | +[default7]: iteration 1146/ 3814 | consumed samples: 586752 | consumed tokens: 1201668096 | elapsed time per iteration (s): 5.44 | learning rate: 8.894E-04 | global batch size: 512 | lm loss: 3.728817E+00 | loss scale: 16384.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.095 | TFLOPs: 71.07 | +[default7]: iteration 1147/ 3814 | consumed samples: 587264 | consumed tokens: 1202716672 | elapsed time per iteration (s): 5.43 | learning rate: 8.891E-04 | global batch size: 512 | lm loss: 3.671301E+00 | loss scale: 16384.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.254 | TFLOPs: 71.19 | +[default7]: iteration 1148/ 3814 | consumed samples: 587776 | consumed tokens: 1203765248 | elapsed time per iteration (s): 5.41 | learning rate: 8.888E-04 | global batch size: 512 | lm loss: 3.678999E+00 | loss scale: 16384.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.556 | TFLOPs: 71.42 | +[default7]: iteration 1149/ 3814 | consumed samples: 588288 | consumed tokens: 1204813824 | elapsed time per iteration (s): 5.43 | learning rate: 8.886E-04 | global batch size: 512 | lm loss: 3.690499E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.327 | TFLOPs: 71.25 | +[default7]: iteration 1150/ 3814 | consumed samples: 588800 | consumed tokens: 1205862400 | elapsed time per iteration (s): 5.40 | learning rate: 8.883E-04 | global batch size: 512 | lm loss: 3.669590E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.883 | TFLOPs: 71.67 | +[default7]: iteration 1151/ 3814 | consumed samples: 589312 | consumed tokens: 1206910976 | elapsed time per iteration (s): 5.38 | learning rate: 8.880E-04 | global batch size: 512 | lm loss: 3.665114E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.131 | TFLOPs: 71.85 | +[default7]: iteration 1152/ 3814 | consumed samples: 589824 | consumed tokens: 1207959552 | elapsed time per iteration (s): 5.38 | learning rate: 8.878E-04 | global batch size: 512 | lm loss: 3.722161E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.255 | TFLOPs: 71.95 | +[default7]: iteration 1153/ 3814 | consumed samples: 590336 | consumed tokens: 1209008128 | elapsed time per iteration (s): 5.42 | learning rate: 8.875E-04 | global batch size: 512 | lm loss: 3.665353E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.509 | TFLOPs: 71.38 | +[default7]: iteration 1154/ 3814 | consumed samples: 590848 | consumed tokens: 1210056704 | elapsed time per iteration (s): 5.41 | learning rate: 8.872E-04 | global batch size: 512 | lm loss: 3.703438E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.559 | TFLOPs: 71.42 | +[default7]: iteration 1155/ 3814 | consumed samples: 591360 | consumed tokens: 1211105280 | elapsed time per iteration (s): 5.42 | learning rate: 8.869E-04 | global batch size: 512 | lm loss: 3.675385E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.500 | TFLOPs: 71.38 | +[default7]: iteration 1156/ 3814 | consumed samples: 591872 | consumed tokens: 1212153856 | elapsed time per iteration (s): 5.41 | learning rate: 8.867E-04 | global batch size: 512 | lm loss: 3.685847E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.724 | TFLOPs: 71.55 | +[default7]: iteration 1157/ 3814 | consumed samples: 592384 | consumed tokens: 1213202432 | elapsed time per iteration (s): 5.42 | learning rate: 8.864E-04 | global batch size: 512 | lm loss: 3.726619E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.521 | TFLOPs: 71.39 | +[default7]: iteration 1158/ 3814 | consumed samples: 592896 | consumed tokens: 1214251008 | elapsed time per iteration (s): 5.40 | learning rate: 8.861E-04 | global batch size: 512 | lm loss: 3.669500E+00 | loss scale: 16384.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.871 | TFLOPs: 71.66 | +[default7]: iteration 1159/ 3814 | consumed samples: 593408 | consumed tokens: 1215299584 | elapsed time per iteration (s): 5.39 | learning rate: 8.859E-04 | global batch size: 512 | lm loss: 3.702272E+00 | loss scale: 16384.0 | grad norm: 0.141 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.050 | TFLOPs: 71.79 | +[default7]: iteration 1160/ 3814 | consumed samples: 593920 | consumed tokens: 1216348160 | elapsed time per iteration (s): 5.40 | learning rate: 8.856E-04 | global batch size: 512 | lm loss: 3.707025E+00 | loss scale: 16384.0 | grad norm: 0.132 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.872 | TFLOPs: 71.66 | +[default7]: iteration 1161/ 3814 | consumed samples: 594432 | consumed tokens: 1217396736 | elapsed time per iteration (s): 5.41 | learning rate: 8.853E-04 | global batch size: 512 | lm loss: 3.707820E+00 | loss scale: 16384.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.613 | TFLOPs: 71.46 | +[default7]: iteration 1162/ 3814 | consumed samples: 594944 | consumed tokens: 1218445312 | elapsed time per iteration (s): 5.40 | learning rate: 8.850E-04 | global batch size: 512 | lm loss: 3.734309E+00 | loss scale: 16384.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.868 | TFLOPs: 71.66 | +[default7]: iteration 1163/ 3814 | consumed samples: 595456 | consumed tokens: 1219493888 | elapsed time per iteration (s): 5.44 | learning rate: 8.848E-04 | global batch size: 512 | lm loss: 3.728378E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.067 | TFLOPs: 71.05 | +[default7]: iteration 1164/ 3814 | consumed samples: 595968 | consumed tokens: 1220542464 | elapsed time per iteration (s): 5.40 | learning rate: 8.845E-04 | global batch size: 512 | lm loss: 3.685004E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.761 | TFLOPs: 71.57 | +[default7]: iteration 1165/ 3814 | consumed samples: 596480 | consumed tokens: 1221591040 | elapsed time per iteration (s): 5.41 | learning rate: 8.842E-04 | global batch size: 512 | lm loss: 3.688124E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.679 | TFLOPs: 71.51 | +[default7]: iteration 1166/ 3814 | consumed samples: 596992 | consumed tokens: 1222639616 | elapsed time per iteration (s): 5.41 | learning rate: 8.839E-04 | global batch size: 512 | lm loss: 3.642985E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.582 | TFLOPs: 71.44 | +[default7]: iteration 1167/ 3814 | consumed samples: 597504 | consumed tokens: 1223688192 | elapsed time per iteration (s): 5.42 | learning rate: 8.837E-04 | global batch size: 512 | lm loss: 3.721063E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1168/ 3814 | consumed samples: 598016 | consumed tokens: 1224736768 | elapsed time per iteration (s): 5.42 | learning rate: 8.834E-04 | global batch size: 512 | lm loss: 3.693168E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.395 | TFLOPs: 71.30 | +[default7]: iteration 1169/ 3814 | consumed samples: 598528 | consumed tokens: 1225785344 | elapsed time per iteration (s): 5.41 | learning rate: 8.831E-04 | global batch size: 512 | lm loss: 3.690622E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.675 | TFLOPs: 71.51 | +[default7]: iteration 1170/ 3814 | consumed samples: 599040 | consumed tokens: 1226833920 | elapsed time per iteration (s): 5.42 | learning rate: 8.828E-04 | global batch size: 512 | lm loss: 3.697329E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.423 | TFLOPs: 71.32 | +[default7]: iteration 1171/ 3814 | consumed samples: 599552 | consumed tokens: 1227882496 | elapsed time per iteration (s): 5.42 | learning rate: 8.826E-04 | global batch size: 512 | lm loss: 3.685559E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.485 | TFLOPs: 71.37 | +[default7]: iteration 1172/ 3814 | consumed samples: 600064 | consumed tokens: 1228931072 | elapsed time per iteration (s): 5.42 | learning rate: 8.823E-04 | global batch size: 512 | lm loss: 3.692276E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 1173/ 3814 | consumed samples: 600576 | consumed tokens: 1229979648 | elapsed time per iteration (s): 5.42 | learning rate: 8.820E-04 | global batch size: 512 | lm loss: 3.704967E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 1174/ 3814 | consumed samples: 601088 | consumed tokens: 1231028224 | elapsed time per iteration (s): 5.41 | learning rate: 8.817E-04 | global batch size: 512 | lm loss: 3.694996E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1175/ 3814 | consumed samples: 601600 | consumed tokens: 1232076800 | elapsed time per iteration (s): 5.42 | learning rate: 8.815E-04 | global batch size: 512 | lm loss: 3.641371E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.507 | TFLOPs: 71.38 | +[default7]: iteration 1176/ 3814 | consumed samples: 602112 | consumed tokens: 1233125376 | elapsed time per iteration (s): 5.41 | learning rate: 8.812E-04 | global batch size: 512 | lm loss: 3.640796E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.584 | TFLOPs: 71.44 | +[default7]: iteration 1177/ 3814 | consumed samples: 602624 | consumed tokens: 1234173952 | elapsed time per iteration (s): 5.43 | learning rate: 8.809E-04 | global batch size: 512 | lm loss: 3.677408E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.353 | TFLOPs: 71.27 | +[default7]: iteration 1178/ 3814 | consumed samples: 603136 | consumed tokens: 1235222528 | elapsed time per iteration (s): 5.41 | learning rate: 8.806E-04 | global batch size: 512 | lm loss: 3.667111E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 1179/ 3814 | consumed samples: 603648 | consumed tokens: 1236271104 | elapsed time per iteration (s): 5.40 | learning rate: 8.804E-04 | global batch size: 512 | lm loss: 3.693630E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.807 | TFLOPs: 71.61 | +[default7]: iteration 1180/ 3814 | consumed samples: 604160 | consumed tokens: 1237319680 | elapsed time per iteration (s): 5.40 | learning rate: 8.801E-04 | global batch size: 512 | lm loss: 3.688366E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.839 | TFLOPs: 71.63 | +[default7]: iteration 1181/ 3814 | consumed samples: 604672 | consumed tokens: 1238368256 | elapsed time per iteration (s): 5.39 | learning rate: 8.798E-04 | global batch size: 512 | lm loss: 3.669701E+00 | loss scale: 16384.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.966 | TFLOPs: 71.73 | +[default7]: iteration 1182/ 3814 | consumed samples: 605184 | consumed tokens: 1239416832 | elapsed time per iteration (s): 5.40 | learning rate: 8.795E-04 | global batch size: 512 | lm loss: 3.700817E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.853 | TFLOPs: 71.64 | +[default7]: iteration 1183/ 3814 | consumed samples: 605696 | consumed tokens: 1240465408 | elapsed time per iteration (s): 5.39 | learning rate: 8.792E-04 | global batch size: 512 | lm loss: 3.666621E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.014 | TFLOPs: 71.77 | +[default7]: iteration 1184/ 3814 | consumed samples: 606208 | consumed tokens: 1241513984 | elapsed time per iteration (s): 5.42 | learning rate: 8.790E-04 | global batch size: 512 | lm loss: 3.695436E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.500 | TFLOPs: 71.38 | +[default7]: iteration 1185/ 3814 | consumed samples: 606720 | consumed tokens: 1242562560 | elapsed time per iteration (s): 5.40 | learning rate: 8.787E-04 | global batch size: 512 | lm loss: 3.660626E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.890 | TFLOPs: 71.67 | +[default7]: iteration 1186/ 3814 | consumed samples: 607232 | consumed tokens: 1243611136 | elapsed time per iteration (s): 5.39 | learning rate: 8.784E-04 | global batch size: 512 | lm loss: 3.658394E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.962 | TFLOPs: 71.73 | +[default7]: iteration 1187/ 3814 | consumed samples: 607744 | consumed tokens: 1244659712 | elapsed time per iteration (s): 5.40 | learning rate: 8.781E-04 | global batch size: 512 | lm loss: 3.675598E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.812 | TFLOPs: 71.61 | +[default7]: iteration 1188/ 3814 | consumed samples: 608256 | consumed tokens: 1245708288 | elapsed time per iteration (s): 5.40 | learning rate: 8.778E-04 | global batch size: 512 | lm loss: 3.652905E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.752 | TFLOPs: 71.57 | +[default7]: iteration 1189/ 3814 | consumed samples: 608768 | consumed tokens: 1246756864 | elapsed time per iteration (s): 5.40 | learning rate: 8.776E-04 | global batch size: 512 | lm loss: 3.696691E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.853 | TFLOPs: 71.64 | +[default7]: iteration 1190/ 3814 | consumed samples: 609280 | consumed tokens: 1247805440 | elapsed time per iteration (s): 5.40 | learning rate: 8.773E-04 | global batch size: 512 | lm loss: 3.653835E+00 | loss scale: 16384.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 1191/ 3814 | consumed samples: 609792 | consumed tokens: 1248854016 | elapsed time per iteration (s): 5.41 | learning rate: 8.770E-04 | global batch size: 512 | lm loss: 3.681258E+00 | loss scale: 16384.0 | grad norm: 0.066 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 1192/ 3814 | consumed samples: 610304 | consumed tokens: 1249902592 | elapsed time per iteration (s): 5.42 | learning rate: 8.767E-04 | global batch size: 512 | lm loss: 3.653280E+00 | loss scale: 16384.0 | grad norm: 0.066 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.416 | TFLOPs: 71.31 | +[default7]: iteration 1193/ 3814 | consumed samples: 610816 | consumed tokens: 1250951168 | elapsed time per iteration (s): 5.42 | learning rate: 8.764E-04 | global batch size: 512 | lm loss: 3.664047E+00 | loss scale: 16384.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.540 | TFLOPs: 71.41 | +[default7]: iteration 1194/ 3814 | consumed samples: 611328 | consumed tokens: 1251999744 | elapsed time per iteration (s): 5.41 | learning rate: 8.762E-04 | global batch size: 512 | lm loss: 3.640771E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.686 | TFLOPs: 71.52 | +[default7]: iteration 1195/ 3814 | consumed samples: 611840 | consumed tokens: 1253048320 | elapsed time per iteration (s): 5.42 | learning rate: 8.759E-04 | global batch size: 512 | lm loss: 3.645024E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 1196/ 3814 | consumed samples: 612352 | consumed tokens: 1254096896 | elapsed time per iteration (s): 5.42 | learning rate: 8.756E-04 | global batch size: 512 | lm loss: 3.656347E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.515 | TFLOPs: 71.39 | +[default7]: iteration 1197/ 3814 | consumed samples: 612864 | consumed tokens: 1255145472 | elapsed time per iteration (s): 5.41 | learning rate: 8.753E-04 | global batch size: 512 | lm loss: 3.655522E+00 | loss scale: 16384.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.644 | TFLOPs: 71.49 | +[default7]: iteration 1198/ 3814 | consumed samples: 613376 | consumed tokens: 1256194048 | elapsed time per iteration (s): 5.43 | learning rate: 8.750E-04 | global batch size: 512 | lm loss: 3.679851E+00 | loss scale: 16384.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.239 | TFLOPs: 71.18 | +[default7]: iteration 1199/ 3814 | consumed samples: 613888 | consumed tokens: 1257242624 | elapsed time per iteration (s): 5.41 | learning rate: 8.747E-04 | global batch size: 512 | lm loss: 3.657740E+00 | loss scale: 16384.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.703 | TFLOPs: 71.53 | +[default7]: iteration 1200/ 3814 | consumed samples: 614400 | consumed tokens: 1258291200 | elapsed time per iteration (s): 5.41 | learning rate: 8.745E-04 | global batch size: 512 | lm loss: 3.660063E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 1201/ 3814 | consumed samples: 614912 | consumed tokens: 1259339776 | elapsed time per iteration (s): 5.43 | learning rate: 8.742E-04 | global batch size: 512 | lm loss: 3.641134E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.376 | TFLOPs: 71.28 | +[default7]: iteration 1202/ 3814 | consumed samples: 615424 | consumed tokens: 1260388352 | elapsed time per iteration (s): 5.43 | learning rate: 8.739E-04 | global batch size: 512 | lm loss: 3.693001E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.226 | TFLOPs: 71.17 | +[default7]: iteration 1203/ 3814 | consumed samples: 615936 | consumed tokens: 1261436928 | elapsed time per iteration (s): 5.43 | learning rate: 8.736E-04 | global batch size: 512 | lm loss: 3.686944E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.290 | TFLOPs: 71.22 | +[default7]: iteration 1204/ 3814 | consumed samples: 616448 | consumed tokens: 1262485504 | elapsed time per iteration (s): 5.43 | learning rate: 8.733E-04 | global batch size: 512 | lm loss: 3.681252E+00 | loss scale: 16384.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.357 | TFLOPs: 71.27 | +[default7]: iteration 1205/ 3814 | consumed samples: 616960 | consumed tokens: 1263534080 | elapsed time per iteration (s): 5.42 | learning rate: 8.730E-04 | global batch size: 512 | lm loss: 3.638278E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.479 | TFLOPs: 71.36 | +[default7]: iteration 1206/ 3814 | consumed samples: 617472 | consumed tokens: 1264582656 | elapsed time per iteration (s): 5.44 | learning rate: 8.728E-04 | global batch size: 512 | lm loss: 3.677603E+00 | loss scale: 16384.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.180 | TFLOPs: 71.14 | +[default7]: iteration 1207/ 3814 | consumed samples: 617984 | consumed tokens: 1265631232 | elapsed time per iteration (s): 5.42 | learning rate: 8.725E-04 | global batch size: 512 | lm loss: 3.655530E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.511 | TFLOPs: 71.38 | +[default7]: iteration 1208/ 3814 | consumed samples: 618496 | consumed tokens: 1266679808 | elapsed time per iteration (s): 5.42 | learning rate: 8.722E-04 | global batch size: 512 | lm loss: 3.629179E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.481 | TFLOPs: 71.36 | +[default7]: iteration 1209/ 3814 | consumed samples: 619008 | consumed tokens: 1267728384 | elapsed time per iteration (s): 5.43 | learning rate: 8.719E-04 | global batch size: 512 | lm loss: 3.655949E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.313 | TFLOPs: 71.24 | +[default7]: iteration 1210/ 3814 | consumed samples: 619520 | consumed tokens: 1268776960 | elapsed time per iteration (s): 5.41 | learning rate: 8.716E-04 | global batch size: 512 | lm loss: 3.641667E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.692 | TFLOPs: 71.52 | +[default7]: iteration 1211/ 3814 | consumed samples: 620032 | consumed tokens: 1269825536 | elapsed time per iteration (s): 5.42 | learning rate: 8.713E-04 | global batch size: 512 | lm loss: 3.632299E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.452 | TFLOPs: 71.34 | +[default7]: iteration 1212/ 3814 | consumed samples: 620544 | consumed tokens: 1270874112 | elapsed time per iteration (s): 5.43 | learning rate: 8.710E-04 | global batch size: 512 | lm loss: 3.622617E+00 | loss scale: 16384.0 | grad norm: 0.069 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.331 | TFLOPs: 71.25 | +[default7]: iteration 1213/ 3814 | consumed samples: 621056 | consumed tokens: 1271922688 | elapsed time per iteration (s): 5.43 | learning rate: 8.708E-04 | global batch size: 512 | lm loss: 3.639251E+00 | loss scale: 16384.0 | grad norm: 0.068 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.323 | TFLOPs: 71.24 | +[default7]: iteration 1214/ 3814 | consumed samples: 621568 | consumed tokens: 1272971264 | elapsed time per iteration (s): 5.43 | learning rate: 8.705E-04 | global batch size: 512 | lm loss: 3.630625E+00 | loss scale: 16384.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.278 | TFLOPs: 71.21 | +[default7]: iteration 1215/ 3814 | consumed samples: 622080 | consumed tokens: 1274019840 | elapsed time per iteration (s): 5.42 | learning rate: 8.702E-04 | global batch size: 512 | lm loss: 3.642107E+00 | loss scale: 16384.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 1216/ 3814 | consumed samples: 622592 | consumed tokens: 1275068416 | elapsed time per iteration (s): 5.42 | learning rate: 8.699E-04 | global batch size: 512 | lm loss: 3.652883E+00 | loss scale: 16384.0 | grad norm: 0.075 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 1217/ 3814 | consumed samples: 623104 | consumed tokens: 1276116992 | elapsed time per iteration (s): 5.43 | learning rate: 8.696E-04 | global batch size: 512 | lm loss: 3.625014E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.361 | TFLOPs: 71.27 | +[default7]: iteration 1218/ 3814 | consumed samples: 623616 | consumed tokens: 1277165568 | elapsed time per iteration (s): 5.43 | learning rate: 8.693E-04 | global batch size: 512 | lm loss: 3.657787E+00 | loss scale: 16384.0 | grad norm: 0.073 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.315 | TFLOPs: 71.24 | +[default7]: iteration 1219/ 3814 | consumed samples: 624128 | consumed tokens: 1278214144 | elapsed time per iteration (s): 5.44 | learning rate: 8.690E-04 | global batch size: 512 | lm loss: 3.621921E+00 | loss scale: 16384.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.090 | TFLOPs: 71.07 | +[default7]: iteration 1220/ 3814 | consumed samples: 624640 | consumed tokens: 1279262720 | elapsed time per iteration (s): 5.42 | learning rate: 8.687E-04 | global batch size: 512 | lm loss: 3.639553E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.423 | TFLOPs: 71.32 | +[default7]: iteration 1221/ 3814 | consumed samples: 625152 | consumed tokens: 1280311296 | elapsed time per iteration (s): 5.41 | learning rate: 8.685E-04 | global batch size: 512 | lm loss: 3.666265E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.599 | TFLOPs: 71.45 | +[default7]: iteration 1222/ 3814 | consumed samples: 625664 | consumed tokens: 1281359872 | elapsed time per iteration (s): 5.42 | learning rate: 8.682E-04 | global batch size: 512 | lm loss: 3.635932E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.497 | TFLOPs: 71.37 | +[default7]: iteration 1223/ 3814 | consumed samples: 626176 | consumed tokens: 1282408448 | elapsed time per iteration (s): 5.41 | learning rate: 8.679E-04 | global batch size: 512 | lm loss: 3.627955E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.48 | +[default7]: iteration 1224/ 3814 | consumed samples: 626688 | consumed tokens: 1283457024 | elapsed time per iteration (s): 5.39 | learning rate: 8.676E-04 | global batch size: 512 | lm loss: 3.655207E+00 | loss scale: 16384.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.942 | TFLOPs: 71.71 | +[default7]: iteration 1225/ 3814 | consumed samples: 627200 | consumed tokens: 1284505600 | elapsed time per iteration (s): 5.40 | learning rate: 8.673E-04 | global batch size: 512 | lm loss: 3.627639E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.775 | TFLOPs: 71.58 | +[default7]: iteration 1226/ 3814 | consumed samples: 627712 | consumed tokens: 1285554176 | elapsed time per iteration (s): 5.42 | learning rate: 8.670E-04 | global batch size: 512 | lm loss: 3.639743E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.543 | TFLOPs: 71.41 | +[default7]: iteration 1227/ 3814 | consumed samples: 628224 | consumed tokens: 1286602752 | elapsed time per iteration (s): 5.41 | learning rate: 8.667E-04 | global batch size: 512 | lm loss: 3.656970E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.678 | TFLOPs: 71.51 | +[default7]: iteration 1228/ 3814 | consumed samples: 628736 | consumed tokens: 1287651328 | elapsed time per iteration (s): 5.39 | learning rate: 8.664E-04 | global batch size: 512 | lm loss: 3.623460E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.955 | TFLOPs: 71.72 | +[default7]: iteration 1229/ 3814 | consumed samples: 629248 | consumed tokens: 1288699904 | elapsed time per iteration (s): 5.41 | learning rate: 8.661E-04 | global batch size: 512 | lm loss: 3.648745E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.563 | TFLOPs: 71.42 | +[default7]: iteration 1230/ 3814 | consumed samples: 629760 | consumed tokens: 1289748480 | elapsed time per iteration (s): 5.40 | learning rate: 8.658E-04 | global batch size: 512 | lm loss: 3.643482E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.773 | TFLOPs: 71.58 | +[default7]: iteration 1231/ 3814 | consumed samples: 630272 | consumed tokens: 1290797056 | elapsed time per iteration (s): 5.42 | learning rate: 8.656E-04 | global batch size: 512 | lm loss: 3.628115E+00 | loss scale: 16384.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.484 | TFLOPs: 71.36 | +[default7]: iteration 1232/ 3814 | consumed samples: 630784 | consumed tokens: 1291845632 | elapsed time per iteration (s): 5.43 | learning rate: 8.653E-04 | global batch size: 512 | lm loss: 3.649860E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 1233/ 3814 | consumed samples: 631296 | consumed tokens: 1292894208 | elapsed time per iteration (s): 5.39 | learning rate: 8.650E-04 | global batch size: 512 | lm loss: 3.639800E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.946 | TFLOPs: 71.71 | +[default7]: iteration 1234/ 3814 | consumed samples: 631808 | consumed tokens: 1293942784 | elapsed time per iteration (s): 5.40 | learning rate: 8.647E-04 | global batch size: 512 | lm loss: 3.615703E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.845 | TFLOPs: 71.64 | +[default7]: iteration 1235/ 3814 | consumed samples: 632320 | consumed tokens: 1294991360 | elapsed time per iteration (s): 5.41 | learning rate: 8.644E-04 | global batch size: 512 | lm loss: 3.627512E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 1236/ 3814 | consumed samples: 632832 | consumed tokens: 1296039936 | elapsed time per iteration (s): 5.38 | learning rate: 8.641E-04 | global batch size: 512 | lm loss: 3.640190E+00 | loss scale: 16384.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.118 | TFLOPs: 71.84 | +[default7]: iteration 1237/ 3814 | consumed samples: 633344 | consumed tokens: 1297088512 | elapsed time per iteration (s): 5.40 | learning rate: 8.638E-04 | global batch size: 512 | lm loss: 3.619972E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.878 | TFLOPs: 71.66 | +[default7]: iteration 1238/ 3814 | consumed samples: 633856 | consumed tokens: 1298137088 | elapsed time per iteration (s): 5.40 | learning rate: 8.635E-04 | global batch size: 512 | lm loss: 3.638139E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.805 | TFLOPs: 71.61 | +[default7]: iteration 1239/ 3814 | consumed samples: 634368 | consumed tokens: 1299185664 | elapsed time per iteration (s): 5.41 | learning rate: 8.632E-04 | global batch size: 512 | lm loss: 3.607861E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.621 | TFLOPs: 71.47 | +[default7]: iteration 1240/ 3814 | consumed samples: 634880 | consumed tokens: 1300234240 | elapsed time per iteration (s): 5.41 | learning rate: 8.629E-04 | global batch size: 512 | lm loss: 3.644633E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.723 | TFLOPs: 71.55 | +[default7]: iteration 1241/ 3814 | consumed samples: 635392 | consumed tokens: 1301282816 | elapsed time per iteration (s): 5.38 | learning rate: 8.626E-04 | global batch size: 512 | lm loss: 3.614863E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.095 | TFLOPs: 71.83 | +[default7]: iteration 1242/ 3814 | consumed samples: 635904 | consumed tokens: 1302331392 | elapsed time per iteration (s): 5.39 | learning rate: 8.623E-04 | global batch size: 512 | lm loss: 3.634840E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.972 | TFLOPs: 71.73 | +[default7]: iteration 1243/ 3814 | consumed samples: 636416 | consumed tokens: 1303379968 | elapsed time per iteration (s): 5.41 | learning rate: 8.620E-04 | global batch size: 512 | lm loss: 3.662102E+00 | loss scale: 16384.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.656 | TFLOPs: 71.49 | +[default7]: iteration 1244/ 3814 | consumed samples: 636928 | consumed tokens: 1304428544 | elapsed time per iteration (s): 5.40 | learning rate: 8.617E-04 | global batch size: 512 | lm loss: 3.654476E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.832 | TFLOPs: 71.63 | +[default7]: iteration 1245/ 3814 | consumed samples: 637440 | consumed tokens: 1305477120 | elapsed time per iteration (s): 5.39 | learning rate: 8.614E-04 | global batch size: 512 | lm loss: 3.662193E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.963 | TFLOPs: 71.73 | +[default7]: iteration 1246/ 3814 | consumed samples: 637952 | consumed tokens: 1306525696 | elapsed time per iteration (s): 5.40 | learning rate: 8.612E-04 | global batch size: 512 | lm loss: 3.634822E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.780 | TFLOPs: 71.59 | +[default7]: iteration 1247/ 3814 | consumed samples: 638464 | consumed tokens: 1307574272 | elapsed time per iteration (s): 5.40 | learning rate: 8.609E-04 | global batch size: 512 | lm loss: 3.603649E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.844 | TFLOPs: 71.64 | +[default7]: iteration 1248/ 3814 | consumed samples: 638976 | consumed tokens: 1308622848 | elapsed time per iteration (s): 5.38 | learning rate: 8.606E-04 | global batch size: 512 | lm loss: 3.649533E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.149 | TFLOPs: 71.87 | +[default7]: iteration 1249/ 3814 | consumed samples: 639488 | consumed tokens: 1309671424 | elapsed time per iteration (s): 5.43 | learning rate: 8.603E-04 | global batch size: 512 | lm loss: 3.639594E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.297 | TFLOPs: 71.22 | +[default0]:saving checkpoint at iteration 1250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 08:12:39,192] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1250 is begin to save! +[default0]:[2023-02-16 08:12:39,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_01-model_00-model_states.pt... +[default7]: iteration 1250/ 3814 | consumed samples: 640000 | consumed tokens: 1310720000 | elapsed time per iteration (s): 5.42 | learning rate: 8.600E-04 | global batch size: 512 | lm loss: 3.591987E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default1]:[2023-02-16 08:12:39,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_01-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,351] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,351] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:39,426] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,427] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,427] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,428] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_05-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:39,498] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,499] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_06-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:39,497] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,498] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,557] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,562] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,562] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_07-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:39,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:39,608] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,609] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,664] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,673] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_09-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:39,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_09-model_00-model_states.pt. +[default1]:[2023-02-16 08:12:39,718] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,718] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,771] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,725] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:39,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,778] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:39,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,824] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,890] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_12-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:39,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,892] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_13-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:39,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,941] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_14-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:39,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:39,993] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:39,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:39,948] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_14-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,000] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,001] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,046] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,053] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,054] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_16-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,107] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,098] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,099] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,152] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,159] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,213] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,205] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,257] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,258] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,266] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_20-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,311] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,312] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,364] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,364] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,372] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,425] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,425] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_23-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,469] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,470] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,478] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,522] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,523] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_25-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,576] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,577] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,530] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,531] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,583] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,584] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,628] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,629] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_27-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,681] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,681] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,636] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,689] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,690] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,736] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,790] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,791] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,743] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_29-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,800] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_30-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,844] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,844] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,854] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_31-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:40,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,909] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:40,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,899] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:40,951] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:40,952] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:40,961] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:40,962] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_33-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:41,017] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:41,017] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 08:12:41,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:41,005] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:41,070] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:41,071] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_35-model_01-model_states.pt... +[default0]:[2023-02-16 08:12:41,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:41,084] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_35-model_00-model_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:41,121] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 08:12:41,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 08:12:41,123] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_01_model_states.pt +[default1]:[2023-02-16 08:12:41,123] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 08:12:41,125] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:12:41,136] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:41,137] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 08:12:41,137] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 08:12:41,139] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_00_model_states.pt +[default0]:[2023-02-16 08:12:41,139] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 08:12:41,141] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:12:41,172] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:12:41,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,672] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,672] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,692] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,692] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,692] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,696] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,697] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,697] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,666] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,666] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,702] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,702] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,670] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,670] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,670] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,706] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,706] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,652] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,652] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,689] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,689] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,690] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,663] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,663] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,663] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,772] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,772] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,723] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,723] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,699] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,700] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,765] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,765] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,797] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,797] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,743] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,743] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,751] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,751] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,759] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,759] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,753] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,753] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,757] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,757] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,771] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,771] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,777] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,777] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,770] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,770] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,793] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,793] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,793] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,775] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,775] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,770] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,770] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,813] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,815] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,815] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,756] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,756] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,817] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,817] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,774] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,774] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,806] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,806] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,806] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,810] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,810] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,768] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,768] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,746] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,746] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,752] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,752] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,778] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,778] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,778] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,802] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,802] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,850] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,850] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,817] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,817] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,840] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,840] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,836] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,836] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,826] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,826] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,805] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,805] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,800] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,800] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,792] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,792] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,792] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,799] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,799] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,799] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:41,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:41,811] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:41,811] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,820] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,820] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:41,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:41,810] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:41,810] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,838] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,838] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:41,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:41,911] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:41,911] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default2]:[2023-02-16 08:12:41,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:12:41,879] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:12:41,879] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,853] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,853] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,941] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,941] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default1]:[2023-02-16 08:12:41,938] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:12:41,938] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:12:41,938] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default4]:[2023-02-16 08:12:41,939] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:12:41,939] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:12:41,939] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default3]:[2023-02-16 08:12:41,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:12:41,923] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:12:41,923] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default5]:[2023-02-16 08:12:41,986] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:12:41,986] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:12:41,986] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:42,066] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:42,066] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:42,066] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default6]:[2023-02-16 08:12:42,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:42,071] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:42,071] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]:[2023-02-16 08:12:42,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:12:42,071] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:12:42,071] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:[2023-02-16 08:12:42,152] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:12:42,152] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:12:42,152] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default7]:time (ms) | save-checkpoint: 2961.72 +[default6]:[2023-02-16 08:12:42,150] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:12:42,150] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1250/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:12:42,150] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1250 is ready now! +[default0]: successfully saved checkpoint at iteration 1250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]: iteration 1251/ 3814 | consumed samples: 640512 | consumed tokens: 1311768576 | elapsed time per iteration (s): 8.39 | learning rate: 8.597E-04 | global batch size: 512 | lm loss: 3.639409E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.061 | TFLOPs: 46.12 | +[default7]: iteration 1252/ 3814 | consumed samples: 641024 | consumed tokens: 1312817152 | elapsed time per iteration (s): 5.42 | learning rate: 8.594E-04 | global batch size: 512 | lm loss: 3.623658E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.483 | TFLOPs: 71.36 | +[default7]: iteration 1253/ 3814 | consumed samples: 641536 | consumed tokens: 1313865728 | elapsed time per iteration (s): 5.42 | learning rate: 8.591E-04 | global batch size: 512 | lm loss: 3.640247E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.415 | TFLOPs: 71.31 | +[default7]: iteration 1254/ 3814 | consumed samples: 642048 | consumed tokens: 1314914304 | elapsed time per iteration (s): 5.42 | learning rate: 8.588E-04 | global batch size: 512 | lm loss: 3.660741E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.387 | TFLOPs: 71.29 | +[default7]: iteration 1255/ 3814 | consumed samples: 642560 | consumed tokens: 1315962880 | elapsed time per iteration (s): 5.43 | learning rate: 8.585E-04 | global batch size: 512 | lm loss: 3.613966E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.215 | TFLOPs: 71.16 | +[default7]: iteration 1256/ 3814 | consumed samples: 643072 | consumed tokens: 1317011456 | elapsed time per iteration (s): 5.42 | learning rate: 8.582E-04 | global batch size: 512 | lm loss: 3.596997E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.484 | TFLOPs: 71.36 | +[default7]: iteration 1257/ 3814 | consumed samples: 643584 | consumed tokens: 1318060032 | elapsed time per iteration (s): 5.41 | learning rate: 8.579E-04 | global batch size: 512 | lm loss: 3.638588E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 1258/ 3814 | consumed samples: 644096 | consumed tokens: 1319108608 | elapsed time per iteration (s): 5.42 | learning rate: 8.576E-04 | global batch size: 512 | lm loss: 3.608510E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.427 | TFLOPs: 71.32 | +[default7]: iteration 1259/ 3814 | consumed samples: 644608 | consumed tokens: 1320157184 | elapsed time per iteration (s): 5.39 | learning rate: 8.573E-04 | global batch size: 512 | lm loss: 3.602135E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.977 | TFLOPs: 71.74 | +[default7]: iteration 1260/ 3814 | consumed samples: 645120 | consumed tokens: 1321205760 | elapsed time per iteration (s): 5.42 | learning rate: 8.570E-04 | global batch size: 512 | lm loss: 3.579546E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1261/ 3814 | consumed samples: 645632 | consumed tokens: 1322254336 | elapsed time per iteration (s): 5.40 | learning rate: 8.567E-04 | global batch size: 512 | lm loss: 3.630316E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.736 | TFLOPs: 71.56 | +[default7]: iteration 1262/ 3814 | consumed samples: 646144 | consumed tokens: 1323302912 | elapsed time per iteration (s): 5.40 | learning rate: 8.564E-04 | global batch size: 512 | lm loss: 3.601615E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.848 | TFLOPs: 71.64 | +[default7]: iteration 1263/ 3814 | consumed samples: 646656 | consumed tokens: 1324351488 | elapsed time per iteration (s): 5.40 | learning rate: 8.561E-04 | global batch size: 512 | lm loss: 3.624441E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.875 | TFLOPs: 71.66 | +[default7]: iteration 1264/ 3814 | consumed samples: 647168 | consumed tokens: 1325400064 | elapsed time per iteration (s): 5.43 | learning rate: 8.558E-04 | global batch size: 512 | lm loss: 3.609349E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.377 | TFLOPs: 71.28 | +[default7]: iteration 1265/ 3814 | consumed samples: 647680 | consumed tokens: 1326448640 | elapsed time per iteration (s): 5.41 | learning rate: 8.555E-04 | global batch size: 512 | lm loss: 3.618723E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 1266/ 3814 | consumed samples: 648192 | consumed tokens: 1327497216 | elapsed time per iteration (s): 5.43 | learning rate: 8.552E-04 | global batch size: 512 | lm loss: 3.597605E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.223 | TFLOPs: 71.17 | +[default7]: iteration 1267/ 3814 | consumed samples: 648704 | consumed tokens: 1328545792 | elapsed time per iteration (s): 5.45 | learning rate: 8.549E-04 | global batch size: 512 | lm loss: 3.587724E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.977 | TFLOPs: 70.98 | +[default7]: iteration 1268/ 3814 | consumed samples: 649216 | consumed tokens: 1329594368 | elapsed time per iteration (s): 5.41 | learning rate: 8.546E-04 | global batch size: 512 | lm loss: 3.606839E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.584 | TFLOPs: 71.44 | +[default7]: iteration 1269/ 3814 | consumed samples: 649728 | consumed tokens: 1330642944 | elapsed time per iteration (s): 5.42 | learning rate: 8.543E-04 | global batch size: 512 | lm loss: 3.633971E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.495 | TFLOPs: 71.37 | +[default7]: iteration 1270/ 3814 | consumed samples: 650240 | consumed tokens: 1331691520 | elapsed time per iteration (s): 5.42 | learning rate: 8.540E-04 | global batch size: 512 | lm loss: 3.632484E+00 | loss scale: 16384.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.463 | TFLOPs: 71.35 | +[default7]: iteration 1271/ 3814 | consumed samples: 650752 | consumed tokens: 1332740096 | elapsed time per iteration (s): 5.42 | learning rate: 8.537E-04 | global batch size: 512 | lm loss: 3.594756E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.535 | TFLOPs: 71.40 | +[default7]: iteration 1272/ 3814 | consumed samples: 651264 | consumed tokens: 1333788672 | elapsed time per iteration (s): 5.42 | learning rate: 8.534E-04 | global batch size: 512 | lm loss: 3.607291E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.497 | TFLOPs: 71.37 | +[default7]: iteration 1273/ 3814 | consumed samples: 651776 | consumed tokens: 1334837248 | elapsed time per iteration (s): 5.43 | learning rate: 8.531E-04 | global batch size: 512 | lm loss: 3.622728E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.229 | TFLOPs: 71.17 | +[default7]: iteration 1274/ 3814 | consumed samples: 652288 | consumed tokens: 1335885824 | elapsed time per iteration (s): 5.42 | learning rate: 8.528E-04 | global batch size: 512 | lm loss: 3.617185E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.463 | TFLOPs: 71.35 | +[default7]: iteration 1275/ 3814 | consumed samples: 652800 | consumed tokens: 1336934400 | elapsed time per iteration (s): 5.41 | learning rate: 8.525E-04 | global batch size: 512 | lm loss: 3.601692E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.593 | TFLOPs: 71.45 | +[default7]: iteration 1276/ 3814 | consumed samples: 653312 | consumed tokens: 1337982976 | elapsed time per iteration (s): 5.45 | learning rate: 8.522E-04 | global batch size: 512 | lm loss: 3.616865E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.025 | TFLOPs: 71.02 | +[default7]: iteration 1277/ 3814 | consumed samples: 653824 | consumed tokens: 1339031552 | elapsed time per iteration (s): 5.42 | learning rate: 8.519E-04 | global batch size: 512 | lm loss: 3.589678E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.459 | TFLOPs: 71.35 | +[default7]: iteration 1278/ 3814 | consumed samples: 654336 | consumed tokens: 1340080128 | elapsed time per iteration (s): 5.44 | learning rate: 8.516E-04 | global batch size: 512 | lm loss: 3.589311E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.177 | TFLOPs: 71.13 | +[default7]: iteration 1279/ 3814 | consumed samples: 654848 | consumed tokens: 1341128704 | elapsed time per iteration (s): 5.43 | learning rate: 8.513E-04 | global batch size: 512 | lm loss: 3.593195E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.266 | TFLOPs: 71.20 | +[default7]: iteration 1280/ 3814 | consumed samples: 655360 | consumed tokens: 1342177280 | elapsed time per iteration (s): 5.44 | learning rate: 8.510E-04 | global batch size: 512 | lm loss: 3.599837E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.103 | TFLOPs: 71.08 | +[default7]: iteration 1281/ 3814 | consumed samples: 655872 | consumed tokens: 1343225856 | elapsed time per iteration (s): 5.42 | learning rate: 8.507E-04 | global batch size: 512 | lm loss: 3.607771E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.397 | TFLOPs: 71.30 | +[default7]: iteration 1282/ 3814 | consumed samples: 656384 | consumed tokens: 1344274432 | elapsed time per iteration (s): 5.43 | learning rate: 8.504E-04 | global batch size: 512 | lm loss: 3.623222E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.369 | TFLOPs: 71.28 | +[default7]: iteration 1283/ 3814 | consumed samples: 656896 | consumed tokens: 1345323008 | elapsed time per iteration (s): 5.41 | learning rate: 8.500E-04 | global batch size: 512 | lm loss: 3.585884E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 1284/ 3814 | consumed samples: 657408 | consumed tokens: 1346371584 | elapsed time per iteration (s): 5.43 | learning rate: 8.497E-04 | global batch size: 512 | lm loss: 3.625152E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.337 | TFLOPs: 71.25 | +[default7]: iteration 1285/ 3814 | consumed samples: 657920 | consumed tokens: 1347420160 | elapsed time per iteration (s): 5.41 | learning rate: 8.494E-04 | global batch size: 512 | lm loss: 3.596348E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.609 | TFLOPs: 71.46 | +[default7]: iteration 1286/ 3814 | consumed samples: 658432 | consumed tokens: 1348468736 | elapsed time per iteration (s): 5.41 | learning rate: 8.491E-04 | global batch size: 512 | lm loss: 3.604101E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.567 | TFLOPs: 71.43 | +[default7]: iteration 1287/ 3814 | consumed samples: 658944 | consumed tokens: 1349517312 | elapsed time per iteration (s): 5.42 | learning rate: 8.488E-04 | global batch size: 512 | lm loss: 3.573934E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.517 | TFLOPs: 71.39 | +[default7]: iteration 1288/ 3814 | consumed samples: 659456 | consumed tokens: 1350565888 | elapsed time per iteration (s): 5.42 | learning rate: 8.485E-04 | global batch size: 512 | lm loss: 3.565052E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1289/ 3814 | consumed samples: 659968 | consumed tokens: 1351614464 | elapsed time per iteration (s): 5.43 | learning rate: 8.482E-04 | global batch size: 512 | lm loss: 3.595228E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.373 | TFLOPs: 71.28 | +[default7]: iteration 1290/ 3814 | consumed samples: 660480 | consumed tokens: 1352663040 | elapsed time per iteration (s): 5.48 | learning rate: 8.479E-04 | global batch size: 512 | lm loss: 3.568544E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.515 | TFLOPs: 70.63 | +[default7]: iteration 1291/ 3814 | consumed samples: 660992 | consumed tokens: 1353711616 | elapsed time per iteration (s): 5.42 | learning rate: 8.476E-04 | global batch size: 512 | lm loss: 3.576310E+00 | loss scale: 16384.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.487 | TFLOPs: 71.37 | +[default7]: iteration 1292/ 3814 | consumed samples: 661504 | consumed tokens: 1354760192 | elapsed time per iteration (s): 5.44 | learning rate: 8.473E-04 | global batch size: 512 | lm loss: 3.584624E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.121 | TFLOPs: 71.09 | +[default7]: iteration 1293/ 3814 | consumed samples: 662016 | consumed tokens: 1355808768 | elapsed time per iteration (s): 5.43 | learning rate: 8.470E-04 | global batch size: 512 | lm loss: 3.539294E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.225 | TFLOPs: 71.17 | +[default7]: iteration 1294/ 3814 | consumed samples: 662528 | consumed tokens: 1356857344 | elapsed time per iteration (s): 5.42 | learning rate: 8.467E-04 | global batch size: 512 | lm loss: 3.591411E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.438 | TFLOPs: 71.33 | +[default7]: iteration 1295/ 3814 | consumed samples: 663040 | consumed tokens: 1357905920 | elapsed time per iteration (s): 5.43 | learning rate: 8.464E-04 | global batch size: 512 | lm loss: 3.598627E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.363 | TFLOPs: 71.27 | +[default7]: iteration 1296/ 3814 | consumed samples: 663552 | consumed tokens: 1358954496 | elapsed time per iteration (s): 5.45 | learning rate: 8.461E-04 | global batch size: 512 | lm loss: 3.584978E+00 | loss scale: 16384.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.869 | TFLOPs: 70.90 | +[default7]: iteration 1297/ 3814 | consumed samples: 664064 | consumed tokens: 1360003072 | elapsed time per iteration (s): 5.42 | learning rate: 8.458E-04 | global batch size: 512 | lm loss: 3.587333E+00 | loss scale: 16384.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.487 | TFLOPs: 71.37 | +[default7]: iteration 1298/ 3814 | consumed samples: 664576 | consumed tokens: 1361051648 | elapsed time per iteration (s): 5.43 | learning rate: 8.454E-04 | global batch size: 512 | lm loss: 3.615928E+00 | loss scale: 16384.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.329 | TFLOPs: 71.25 | +[default7]: iteration 1299/ 3814 | consumed samples: 665088 | consumed tokens: 1362100224 | elapsed time per iteration (s): 5.43 | learning rate: 8.451E-04 | global batch size: 512 | lm loss: 3.610089E+00 | loss scale: 16384.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.337 | TFLOPs: 71.25 | +[default7]: iteration 1300/ 3814 | consumed samples: 665600 | consumed tokens: 1363148800 | elapsed time per iteration (s): 5.41 | learning rate: 8.448E-04 | global batch size: 512 | lm loss: 3.568993E+00 | loss scale: 16384.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.722 | TFLOPs: 71.54 | +[default7]: iteration 1301/ 3814 | consumed samples: 666112 | consumed tokens: 1364197376 | elapsed time per iteration (s): 5.44 | learning rate: 8.445E-04 | global batch size: 512 | lm loss: 3.573572E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.155 | TFLOPs: 71.12 | +[default7]: iteration 1302/ 3814 | consumed samples: 666624 | consumed tokens: 1365245952 | elapsed time per iteration (s): 5.40 | learning rate: 8.442E-04 | global batch size: 512 | lm loss: 3.586007E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.756 | TFLOPs: 71.57 | +[default7]: iteration 1303/ 3814 | consumed samples: 667136 | consumed tokens: 1366294528 | elapsed time per iteration (s): 5.40 | learning rate: 8.439E-04 | global batch size: 512 | lm loss: 3.609221E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.804 | TFLOPs: 71.61 | +[default7]: iteration 1304/ 3814 | consumed samples: 667648 | consumed tokens: 1367343104 | elapsed time per iteration (s): 5.41 | learning rate: 8.436E-04 | global batch size: 512 | lm loss: 3.584699E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 1305/ 3814 | consumed samples: 668160 | consumed tokens: 1368391680 | elapsed time per iteration (s): 5.41 | learning rate: 8.433E-04 | global batch size: 512 | lm loss: 3.571339E+00 | loss scale: 16384.0 | grad norm: 0.077 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.711 | TFLOPs: 71.54 | +[default7]: iteration 1306/ 3814 | consumed samples: 668672 | consumed tokens: 1369440256 | elapsed time per iteration (s): 5.42 | learning rate: 8.430E-04 | global batch size: 512 | lm loss: 3.569804E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.548 | TFLOPs: 71.41 | +[default7]: iteration 1307/ 3814 | consumed samples: 669184 | consumed tokens: 1370488832 | elapsed time per iteration (s): 5.43 | learning rate: 8.427E-04 | global batch size: 512 | lm loss: 3.552865E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.208 | TFLOPs: 71.16 | +[default7]: iteration 1308/ 3814 | consumed samples: 669696 | consumed tokens: 1371537408 | elapsed time per iteration (s): 5.41 | learning rate: 8.424E-04 | global batch size: 512 | lm loss: 3.557495E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 1309/ 3814 | consumed samples: 670208 | consumed tokens: 1372585984 | elapsed time per iteration (s): 5.42 | learning rate: 8.420E-04 | global batch size: 512 | lm loss: 3.571532E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1310/ 3814 | consumed samples: 670720 | consumed tokens: 1373634560 | elapsed time per iteration (s): 5.40 | learning rate: 8.417E-04 | global batch size: 512 | lm loss: 3.629792E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.779 | TFLOPs: 71.59 | +[default7]: iteration 1311/ 3814 | consumed samples: 671232 | consumed tokens: 1374683136 | elapsed time per iteration (s): 5.42 | learning rate: 8.414E-04 | global batch size: 512 | lm loss: 3.566921E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 1312/ 3814 | consumed samples: 671744 | consumed tokens: 1375731712 | elapsed time per iteration (s): 5.44 | learning rate: 8.411E-04 | global batch size: 512 | lm loss: 3.576440E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.045 | TFLOPs: 71.03 | +[default7]: iteration 1313/ 3814 | consumed samples: 672256 | consumed tokens: 1376780288 | elapsed time per iteration (s): 5.42 | learning rate: 8.408E-04 | global batch size: 512 | lm loss: 3.601136E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.493 | TFLOPs: 71.37 | +[default7]: iteration 1314/ 3814 | consumed samples: 672768 | consumed tokens: 1377828864 | elapsed time per iteration (s): 5.41 | learning rate: 8.405E-04 | global batch size: 512 | lm loss: 3.588197E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.696 | TFLOPs: 71.53 | +[default7]: iteration 1315/ 3814 | consumed samples: 673280 | consumed tokens: 1378877440 | elapsed time per iteration (s): 5.43 | learning rate: 8.402E-04 | global batch size: 512 | lm loss: 3.567047E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.284 | TFLOPs: 71.21 | +[default7]: iteration 1316/ 3814 | consumed samples: 673792 | consumed tokens: 1379926016 | elapsed time per iteration (s): 5.42 | learning rate: 8.399E-04 | global batch size: 512 | lm loss: 3.544058E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 1317/ 3814 | consumed samples: 674304 | consumed tokens: 1380974592 | elapsed time per iteration (s): 5.43 | learning rate: 8.395E-04 | global batch size: 512 | lm loss: 3.577523E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.331 | TFLOPs: 71.25 | +[default7]: iteration 1318/ 3814 | consumed samples: 674816 | consumed tokens: 1382023168 | elapsed time per iteration (s): 5.42 | learning rate: 8.392E-04 | global batch size: 512 | lm loss: 3.593098E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.413 | TFLOPs: 71.31 | +[default7]: iteration 1319/ 3814 | consumed samples: 675328 | consumed tokens: 1383071744 | elapsed time per iteration (s): 5.42 | learning rate: 8.389E-04 | global batch size: 512 | lm loss: 3.573915E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.420 | TFLOPs: 71.32 | +[default7]: iteration 1320/ 3814 | consumed samples: 675840 | consumed tokens: 1384120320 | elapsed time per iteration (s): 5.40 | learning rate: 8.386E-04 | global batch size: 512 | lm loss: 3.569281E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 1321/ 3814 | consumed samples: 676352 | consumed tokens: 1385168896 | elapsed time per iteration (s): 5.41 | learning rate: 8.383E-04 | global batch size: 512 | lm loss: 3.567278E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default7]: iteration 1322/ 3814 | consumed samples: 676864 | consumed tokens: 1386217472 | elapsed time per iteration (s): 5.43 | learning rate: 8.380E-04 | global batch size: 512 | lm loss: 3.577801E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.277 | TFLOPs: 71.21 | +[default7]: iteration 1323/ 3814 | consumed samples: 677376 | consumed tokens: 1387266048 | elapsed time per iteration (s): 5.42 | learning rate: 8.377E-04 | global batch size: 512 | lm loss: 3.553213E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.526 | TFLOPs: 71.40 | +[default7]: iteration 1324/ 3814 | consumed samples: 677888 | consumed tokens: 1388314624 | elapsed time per iteration (s): 5.43 | learning rate: 8.373E-04 | global batch size: 512 | lm loss: 3.591558E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.349 | TFLOPs: 71.26 | +[default7]: iteration 1325/ 3814 | consumed samples: 678400 | consumed tokens: 1389363200 | elapsed time per iteration (s): 5.43 | learning rate: 8.370E-04 | global batch size: 512 | lm loss: 3.563124E+00 | loss scale: 16384.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 1326/ 3814 | consumed samples: 678912 | consumed tokens: 1390411776 | elapsed time per iteration (s): 5.40 | learning rate: 8.367E-04 | global batch size: 512 | lm loss: 3.599374E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.765 | TFLOPs: 71.58 | +[default7]: iteration 1327/ 3814 | consumed samples: 679424 | consumed tokens: 1391460352 | elapsed time per iteration (s): 5.38 | learning rate: 8.364E-04 | global batch size: 512 | lm loss: 3.570580E+00 | loss scale: 16384.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.090 | TFLOPs: 71.82 | +[default7]: iteration 1328/ 3814 | consumed samples: 679936 | consumed tokens: 1392508928 | elapsed time per iteration (s): 5.40 | learning rate: 8.361E-04 | global batch size: 512 | lm loss: 3.575208E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.773 | TFLOPs: 71.58 | +[default7]: iteration 1329/ 3814 | consumed samples: 680448 | consumed tokens: 1393557504 | elapsed time per iteration (s): 5.38 | learning rate: 8.358E-04 | global batch size: 512 | lm loss: 3.600633E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.107 | TFLOPs: 71.83 | +[default7]: iteration 1330/ 3814 | consumed samples: 680960 | consumed tokens: 1394606080 | elapsed time per iteration (s): 5.40 | learning rate: 8.355E-04 | global batch size: 512 | lm loss: 3.580013E+00 | loss scale: 16384.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 1331/ 3814 | consumed samples: 681472 | consumed tokens: 1395654656 | elapsed time per iteration (s): 5.41 | learning rate: 8.351E-04 | global batch size: 512 | lm loss: 3.584107E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.701 | TFLOPs: 71.53 | +[default7]: iteration 1332/ 3814 | consumed samples: 681984 | consumed tokens: 1396703232 | elapsed time per iteration (s): 5.40 | learning rate: 8.348E-04 | global batch size: 512 | lm loss: 3.565153E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.890 | TFLOPs: 71.67 | +[default7]: iteration 1333/ 3814 | consumed samples: 682496 | consumed tokens: 1397751808 | elapsed time per iteration (s): 5.39 | learning rate: 8.345E-04 | global batch size: 512 | lm loss: 3.531796E+00 | loss scale: 16384.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.982 | TFLOPs: 71.74 | +[default7]: iteration 1334/ 3814 | consumed samples: 683008 | consumed tokens: 1398800384 | elapsed time per iteration (s): 5.42 | learning rate: 8.342E-04 | global batch size: 512 | lm loss: 3.565475E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.520 | TFLOPs: 71.39 | +[default7]: iteration 1335/ 3814 | consumed samples: 683520 | consumed tokens: 1399848960 | elapsed time per iteration (s): 5.42 | learning rate: 8.339E-04 | global batch size: 512 | lm loss: 3.580370E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.482 | TFLOPs: 71.36 | +[default7]: iteration 1336/ 3814 | consumed samples: 684032 | consumed tokens: 1400897536 | elapsed time per iteration (s): 5.42 | learning rate: 8.336E-04 | global batch size: 512 | lm loss: 3.569579E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.438 | TFLOPs: 71.33 | +[default7]: iteration 1337/ 3814 | consumed samples: 684544 | consumed tokens: 1401946112 | elapsed time per iteration (s): 5.41 | learning rate: 8.332E-04 | global batch size: 512 | lm loss: 3.558713E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 1338/ 3814 | consumed samples: 685056 | consumed tokens: 1402994688 | elapsed time per iteration (s): 5.42 | learning rate: 8.329E-04 | global batch size: 512 | lm loss: 3.528993E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.520 | TFLOPs: 71.39 | +[default7]: iteration 1339/ 3814 | consumed samples: 685568 | consumed tokens: 1404043264 | elapsed time per iteration (s): 5.41 | learning rate: 8.326E-04 | global batch size: 512 | lm loss: 3.560404E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.575 | TFLOPs: 71.43 | +[default7]: iteration 1340/ 3814 | consumed samples: 686080 | consumed tokens: 1405091840 | elapsed time per iteration (s): 5.42 | learning rate: 8.323E-04 | global batch size: 512 | lm loss: 3.599029E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 1341/ 3814 | consumed samples: 686592 | consumed tokens: 1406140416 | elapsed time per iteration (s): 5.40 | learning rate: 8.320E-04 | global batch size: 512 | lm loss: 3.567458E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.733 | TFLOPs: 71.55 | +[default7]: iteration 1342/ 3814 | consumed samples: 687104 | consumed tokens: 1407188992 | elapsed time per iteration (s): 5.42 | learning rate: 8.316E-04 | global batch size: 512 | lm loss: 3.551879E+00 | loss scale: 16384.0 | grad norm: 0.076 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 1343/ 3814 | consumed samples: 687616 | consumed tokens: 1408237568 | elapsed time per iteration (s): 5.41 | learning rate: 8.313E-04 | global batch size: 512 | lm loss: 3.560005E+00 | loss scale: 16384.0 | grad norm: 0.072 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 1344/ 3814 | consumed samples: 688128 | consumed tokens: 1409286144 | elapsed time per iteration (s): 5.40 | learning rate: 8.310E-04 | global batch size: 512 | lm loss: 3.534563E+00 | loss scale: 16384.0 | grad norm: 0.071 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 1345/ 3814 | consumed samples: 688640 | consumed tokens: 1410334720 | elapsed time per iteration (s): 5.40 | learning rate: 8.307E-04 | global batch size: 512 | lm loss: 3.550020E+00 | loss scale: 16384.0 | grad norm: 0.074 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.833 | TFLOPs: 71.63 | +[default7]: iteration 1346/ 3814 | consumed samples: 689152 | consumed tokens: 1411383296 | elapsed time per iteration (s): 5.39 | learning rate: 8.304E-04 | global batch size: 512 | lm loss: 3.561210E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.993 | TFLOPs: 71.75 | +[default7]: iteration 1347/ 3814 | consumed samples: 689664 | consumed tokens: 1412431872 | elapsed time per iteration (s): 5.40 | learning rate: 8.300E-04 | global batch size: 512 | lm loss: 3.518404E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 1348/ 3814 | consumed samples: 690176 | consumed tokens: 1413480448 | elapsed time per iteration (s): 5.40 | learning rate: 8.297E-04 | global batch size: 512 | lm loss: 3.571887E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.754 | TFLOPs: 71.57 | +[default7]: iteration 1349/ 3814 | consumed samples: 690688 | consumed tokens: 1414529024 | elapsed time per iteration (s): 5.39 | learning rate: 8.294E-04 | global batch size: 512 | lm loss: 3.558763E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.014 | TFLOPs: 71.77 | +[default7]: iteration 1350/ 3814 | consumed samples: 691200 | consumed tokens: 1415577600 | elapsed time per iteration (s): 5.42 | learning rate: 8.291E-04 | global batch size: 512 | lm loss: 3.546903E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.493 | TFLOPs: 71.37 | +[default7]: iteration 1351/ 3814 | consumed samples: 691712 | consumed tokens: 1416626176 | elapsed time per iteration (s): 5.42 | learning rate: 8.288E-04 | global batch size: 512 | lm loss: 3.584992E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.502 | TFLOPs: 71.38 | +[default7]: iteration 1352/ 3814 | consumed samples: 692224 | consumed tokens: 1417674752 | elapsed time per iteration (s): 5.42 | learning rate: 8.284E-04 | global batch size: 512 | lm loss: 3.557261E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 1353/ 3814 | consumed samples: 692736 | consumed tokens: 1418723328 | elapsed time per iteration (s): 5.42 | learning rate: 8.281E-04 | global batch size: 512 | lm loss: 3.560409E+00 | loss scale: 16384.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.498 | TFLOPs: 71.38 | +[default7]: iteration 1354/ 3814 | consumed samples: 693248 | consumed tokens: 1419771904 | elapsed time per iteration (s): 5.41 | learning rate: 8.278E-04 | global batch size: 512 | lm loss: 3.573054E+00 | loss scale: 16384.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.675 | TFLOPs: 71.51 | +[default7]: iteration 1355/ 3814 | consumed samples: 693760 | consumed tokens: 1420820480 | elapsed time per iteration (s): 5.42 | learning rate: 8.275E-04 | global batch size: 512 | lm loss: 3.561970E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.517 | TFLOPs: 71.39 | +[default7]: iteration 1356/ 3814 | consumed samples: 694272 | consumed tokens: 1421869056 | elapsed time per iteration (s): 5.41 | learning rate: 8.272E-04 | global batch size: 512 | lm loss: 3.550116E+00 | loss scale: 16384.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 1357/ 3814 | consumed samples: 694784 | consumed tokens: 1422917632 | elapsed time per iteration (s): 5.43 | learning rate: 8.268E-04 | global batch size: 512 | lm loss: 3.553225E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.256 | TFLOPs: 71.19 | +[default7]: iteration 1358/ 3814 | consumed samples: 695296 | consumed tokens: 1423966208 | elapsed time per iteration (s): 5.42 | learning rate: 8.265E-04 | global batch size: 512 | lm loss: 3.549227E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.479 | TFLOPs: 71.36 | +[default7]: iteration 1359/ 3814 | consumed samples: 695808 | consumed tokens: 1425014784 | elapsed time per iteration (s): 5.38 | learning rate: 8.262E-04 | global batch size: 512 | lm loss: 3.569834E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.095 | TFLOPs: 71.83 | +[default7]: iteration 1360/ 3814 | consumed samples: 696320 | consumed tokens: 1426063360 | elapsed time per iteration (s): 5.41 | learning rate: 8.259E-04 | global batch size: 512 | lm loss: 3.550525E+00 | loss scale: 16384.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.605 | TFLOPs: 71.46 | +[default7]: iteration 1361/ 3814 | consumed samples: 696832 | consumed tokens: 1427111936 | elapsed time per iteration (s): 5.38 | learning rate: 8.255E-04 | global batch size: 512 | lm loss: 3.560366E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.101 | TFLOPs: 71.83 | +[default7]: iteration 1362/ 3814 | consumed samples: 697344 | consumed tokens: 1428160512 | elapsed time per iteration (s): 5.41 | learning rate: 8.252E-04 | global batch size: 512 | lm loss: 3.550585E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.643 | TFLOPs: 71.49 | +[default7]: iteration 1363/ 3814 | consumed samples: 697856 | consumed tokens: 1429209088 | elapsed time per iteration (s): 5.41 | learning rate: 8.249E-04 | global batch size: 512 | lm loss: 3.536980E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.660 | TFLOPs: 71.50 | +[default7]: iteration 1364/ 3814 | consumed samples: 698368 | consumed tokens: 1430257664 | elapsed time per iteration (s): 5.40 | learning rate: 8.246E-04 | global batch size: 512 | lm loss: 3.551361E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.770 | TFLOPs: 71.58 | +[default7]: iteration 1365/ 3814 | consumed samples: 698880 | consumed tokens: 1431306240 | elapsed time per iteration (s): 5.41 | learning rate: 8.242E-04 | global batch size: 512 | lm loss: 3.559350E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.619 | TFLOPs: 71.47 | +[default7]: iteration 1366/ 3814 | consumed samples: 699392 | consumed tokens: 1432354816 | elapsed time per iteration (s): 5.42 | learning rate: 8.239E-04 | global batch size: 512 | lm loss: 3.552106E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.480 | TFLOPs: 71.36 | +[default7]: iteration 1367/ 3814 | consumed samples: 699904 | consumed tokens: 1433403392 | elapsed time per iteration (s): 5.41 | learning rate: 8.236E-04 | global batch size: 512 | lm loss: 3.533653E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.557 | TFLOPs: 71.42 | +[default7]: iteration 1368/ 3814 | consumed samples: 700416 | consumed tokens: 1434451968 | elapsed time per iteration (s): 5.43 | learning rate: 8.233E-04 | global batch size: 512 | lm loss: 3.561731E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 1369/ 3814 | consumed samples: 700928 | consumed tokens: 1435500544 | elapsed time per iteration (s): 5.41 | learning rate: 8.229E-04 | global batch size: 512 | lm loss: 3.554584E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1370/ 3814 | consumed samples: 701440 | consumed tokens: 1436549120 | elapsed time per iteration (s): 5.44 | learning rate: 8.226E-04 | global batch size: 512 | lm loss: 3.534064E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.121 | TFLOPs: 71.09 | +[default7]: iteration 1371/ 3814 | consumed samples: 701952 | consumed tokens: 1437597696 | elapsed time per iteration (s): 5.42 | learning rate: 8.223E-04 | global batch size: 512 | lm loss: 3.522353E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 1372/ 3814 | consumed samples: 702464 | consumed tokens: 1438646272 | elapsed time per iteration (s): 5.40 | learning rate: 8.220E-04 | global batch size: 512 | lm loss: 3.542519E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.789 | TFLOPs: 71.60 | +[default7]: iteration 1373/ 3814 | consumed samples: 702976 | consumed tokens: 1439694848 | elapsed time per iteration (s): 5.43 | learning rate: 8.216E-04 | global batch size: 512 | lm loss: 3.526170E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.332 | TFLOPs: 71.25 | +[default7]: iteration 1374/ 3814 | consumed samples: 703488 | consumed tokens: 1440743424 | elapsed time per iteration (s): 5.43 | learning rate: 8.213E-04 | global batch size: 512 | lm loss: 3.520628E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.342 | TFLOPs: 71.26 | +[default7]: iteration 1375/ 3814 | consumed samples: 704000 | consumed tokens: 1441792000 | elapsed time per iteration (s): 5.39 | learning rate: 8.210E-04 | global batch size: 512 | lm loss: 3.558412E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.943 | TFLOPs: 71.71 | +[default7]: iteration 1376/ 3814 | consumed samples: 704512 | consumed tokens: 1442840576 | elapsed time per iteration (s): 5.41 | learning rate: 8.207E-04 | global batch size: 512 | lm loss: 3.549283E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.674 | TFLOPs: 71.51 | +[default7]: iteration 1377/ 3814 | consumed samples: 705024 | consumed tokens: 1443889152 | elapsed time per iteration (s): 5.39 | learning rate: 8.203E-04 | global batch size: 512 | lm loss: 3.553836E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.938 | TFLOPs: 71.71 | +[default7]: iteration 1378/ 3814 | consumed samples: 705536 | consumed tokens: 1444937728 | elapsed time per iteration (s): 5.39 | learning rate: 8.200E-04 | global batch size: 512 | lm loss: 3.528210E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.002 | TFLOPs: 71.76 | +[default7]: iteration 1379/ 3814 | consumed samples: 706048 | consumed tokens: 1445986304 | elapsed time per iteration (s): 5.42 | learning rate: 8.197E-04 | global batch size: 512 | lm loss: 3.561156E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.481 | TFLOPs: 71.36 | +[default7]: iteration 1380/ 3814 | consumed samples: 706560 | consumed tokens: 1447034880 | elapsed time per iteration (s): 5.39 | learning rate: 8.194E-04 | global batch size: 512 | lm loss: 3.508819E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.967 | TFLOPs: 71.73 | +[default7]: iteration 1381/ 3814 | consumed samples: 707072 | consumed tokens: 1448083456 | elapsed time per iteration (s): 5.40 | learning rate: 8.190E-04 | global batch size: 512 | lm loss: 3.520690E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.772 | TFLOPs: 71.58 | +[default7]: iteration 1382/ 3814 | consumed samples: 707584 | consumed tokens: 1449132032 | elapsed time per iteration (s): 5.41 | learning rate: 8.187E-04 | global batch size: 512 | lm loss: 3.529475E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.642 | TFLOPs: 71.48 | +[default7]: iteration 1383/ 3814 | consumed samples: 708096 | consumed tokens: 1450180608 | elapsed time per iteration (s): 5.41 | learning rate: 8.184E-04 | global batch size: 512 | lm loss: 3.525594E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.697 | TFLOPs: 71.53 | +[default7]: iteration 1384/ 3814 | consumed samples: 708608 | consumed tokens: 1451229184 | elapsed time per iteration (s): 5.41 | learning rate: 8.180E-04 | global batch size: 512 | lm loss: 3.516468E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.556 | TFLOPs: 71.42 | +[default7]: iteration 1385/ 3814 | consumed samples: 709120 | consumed tokens: 1452277760 | elapsed time per iteration (s): 5.41 | learning rate: 8.177E-04 | global batch size: 512 | lm loss: 3.540309E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 1386/ 3814 | consumed samples: 709632 | consumed tokens: 1453326336 | elapsed time per iteration (s): 5.41 | learning rate: 8.174E-04 | global batch size: 512 | lm loss: 3.528860E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.619 | TFLOPs: 71.47 | +[default7]: iteration 1387/ 3814 | consumed samples: 710144 | consumed tokens: 1454374912 | elapsed time per iteration (s): 5.41 | learning rate: 8.171E-04 | global batch size: 512 | lm loss: 3.558701E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1388/ 3814 | consumed samples: 710656 | consumed tokens: 1455423488 | elapsed time per iteration (s): 5.40 | learning rate: 8.167E-04 | global batch size: 512 | lm loss: 3.515197E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.752 | TFLOPs: 71.57 | +[default7]: iteration 1389/ 3814 | consumed samples: 711168 | consumed tokens: 1456472064 | elapsed time per iteration (s): 5.40 | learning rate: 8.164E-04 | global batch size: 512 | lm loss: 3.562329E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.750 | TFLOPs: 71.57 | +[default7]: iteration 1390/ 3814 | consumed samples: 711680 | consumed tokens: 1457520640 | elapsed time per iteration (s): 5.41 | learning rate: 8.161E-04 | global batch size: 512 | lm loss: 3.522927E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1391/ 3814 | consumed samples: 712192 | consumed tokens: 1458569216 | elapsed time per iteration (s): 5.41 | learning rate: 8.157E-04 | global batch size: 512 | lm loss: 3.529679E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1392/ 3814 | consumed samples: 712704 | consumed tokens: 1459617792 | elapsed time per iteration (s): 5.42 | learning rate: 8.154E-04 | global batch size: 512 | lm loss: 3.506601E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 1393/ 3814 | consumed samples: 713216 | consumed tokens: 1460666368 | elapsed time per iteration (s): 5.40 | learning rate: 8.151E-04 | global batch size: 512 | lm loss: 3.549001E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.843 | TFLOPs: 71.64 | +[default7]: iteration 1394/ 3814 | consumed samples: 713728 | consumed tokens: 1461714944 | elapsed time per iteration (s): 5.43 | learning rate: 8.148E-04 | global batch size: 512 | lm loss: 3.524196E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.241 | TFLOPs: 71.18 | +[default7]: iteration 1395/ 3814 | consumed samples: 714240 | consumed tokens: 1462763520 | elapsed time per iteration (s): 5.42 | learning rate: 8.144E-04 | global batch size: 512 | lm loss: 3.529450E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.395 | TFLOPs: 71.30 | +[default7]: iteration 1396/ 3814 | consumed samples: 714752 | consumed tokens: 1463812096 | elapsed time per iteration (s): 5.43 | learning rate: 8.141E-04 | global batch size: 512 | lm loss: 3.498338E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 1397/ 3814 | consumed samples: 715264 | consumed tokens: 1464860672 | elapsed time per iteration (s): 5.43 | learning rate: 8.138E-04 | global batch size: 512 | lm loss: 3.553322E+00 | loss scale: 16384.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.261 | TFLOPs: 71.20 | +[default7]: iteration 1398/ 3814 | consumed samples: 715776 | consumed tokens: 1465909248 | elapsed time per iteration (s): 5.42 | learning rate: 8.134E-04 | global batch size: 512 | lm loss: 3.541627E+00 | loss scale: 16384.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.515 | TFLOPs: 71.39 | +[default7]: iteration 1399/ 3814 | consumed samples: 716288 | consumed tokens: 1466957824 | elapsed time per iteration (s): 5.43 | learning rate: 8.131E-04 | global batch size: 512 | lm loss: 3.513378E+00 | loss scale: 16384.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.356 | TFLOPs: 71.27 | +[default7]: iteration 1400/ 3814 | consumed samples: 716800 | consumed tokens: 1468006400 | elapsed time per iteration (s): 5.40 | learning rate: 8.128E-04 | global batch size: 512 | lm loss: 3.519351E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.739 | TFLOPs: 71.56 | +[default7]: iteration 1401/ 3814 | consumed samples: 717312 | consumed tokens: 1469054976 | elapsed time per iteration (s): 5.42 | learning rate: 8.124E-04 | global batch size: 512 | lm loss: 3.518397E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.443 | TFLOPs: 71.33 | +[default7]: iteration 1402/ 3814 | consumed samples: 717824 | consumed tokens: 1470103552 | elapsed time per iteration (s): 5.41 | learning rate: 8.121E-04 | global batch size: 512 | lm loss: 3.534405E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.555 | TFLOPs: 71.42 | +[default7]: iteration 1403/ 3814 | consumed samples: 718336 | consumed tokens: 1471152128 | elapsed time per iteration (s): 5.42 | learning rate: 8.118E-04 | global batch size: 512 | lm loss: 3.525983E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 1404/ 3814 | consumed samples: 718848 | consumed tokens: 1472200704 | elapsed time per iteration (s): 5.41 | learning rate: 8.114E-04 | global batch size: 512 | lm loss: 3.518807E+00 | loss scale: 16384.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.701 | TFLOPs: 71.53 | +[default7]: iteration 1405/ 3814 | consumed samples: 719360 | consumed tokens: 1473249280 | elapsed time per iteration (s): 5.41 | learning rate: 8.111E-04 | global batch size: 512 | lm loss: 3.493307E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.565 | TFLOPs: 71.43 | +[default7]: iteration 1406/ 3814 | consumed samples: 719872 | consumed tokens: 1474297856 | elapsed time per iteration (s): 5.41 | learning rate: 8.108E-04 | global batch size: 512 | lm loss: 3.516969E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.581 | TFLOPs: 71.44 | +[default7]: iteration 1407/ 3814 | consumed samples: 720384 | consumed tokens: 1475346432 | elapsed time per iteration (s): 5.42 | learning rate: 8.104E-04 | global batch size: 512 | lm loss: 3.502759E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 1408/ 3814 | consumed samples: 720896 | consumed tokens: 1476395008 | elapsed time per iteration (s): 5.42 | learning rate: 8.101E-04 | global batch size: 512 | lm loss: 3.534981E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.466 | TFLOPs: 71.35 | +[default7]: iteration 1409/ 3814 | consumed samples: 721408 | consumed tokens: 1477443584 | elapsed time per iteration (s): 5.42 | learning rate: 8.098E-04 | global batch size: 512 | lm loss: 3.508100E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.412 | TFLOPs: 71.31 | +[default7]: iteration 1410/ 3814 | consumed samples: 721920 | consumed tokens: 1478492160 | elapsed time per iteration (s): 5.42 | learning rate: 8.094E-04 | global batch size: 512 | lm loss: 3.502895E+00 | loss scale: 16384.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.442 | TFLOPs: 71.33 | +[default7]: iteration 1411/ 3814 | consumed samples: 722432 | consumed tokens: 1479540736 | elapsed time per iteration (s): 5.40 | learning rate: 8.091E-04 | global batch size: 512 | lm loss: 3.527426E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.893 | TFLOPs: 71.67 | +[default7]: iteration 1412/ 3814 | consumed samples: 722944 | consumed tokens: 1480589312 | elapsed time per iteration (s): 5.41 | learning rate: 8.088E-04 | global batch size: 512 | lm loss: 3.514228E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.721 | TFLOPs: 71.54 | +[default7]: iteration 1413/ 3814 | consumed samples: 723456 | consumed tokens: 1481637888 | elapsed time per iteration (s): 5.42 | learning rate: 8.084E-04 | global batch size: 512 | lm loss: 3.525518E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.524 | TFLOPs: 71.39 | +[default7]: iteration 1414/ 3814 | consumed samples: 723968 | consumed tokens: 1482686464 | elapsed time per iteration (s): 5.42 | learning rate: 8.081E-04 | global batch size: 512 | lm loss: 3.521048E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 1415/ 3814 | consumed samples: 724480 | consumed tokens: 1483735040 | elapsed time per iteration (s): 5.40 | learning rate: 8.078E-04 | global batch size: 512 | lm loss: 3.507548E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.754 | TFLOPs: 71.57 | +[default7]: iteration 1416/ 3814 | consumed samples: 724992 | consumed tokens: 1484783616 | elapsed time per iteration (s): 5.43 | learning rate: 8.074E-04 | global batch size: 512 | lm loss: 3.502088E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.323 | TFLOPs: 71.24 | +[default7]: iteration 1417/ 3814 | consumed samples: 725504 | consumed tokens: 1485832192 | elapsed time per iteration (s): 5.42 | learning rate: 8.071E-04 | global batch size: 512 | lm loss: 3.524494E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.458 | TFLOPs: 71.34 | +[default7]: iteration 1418/ 3814 | consumed samples: 726016 | consumed tokens: 1486880768 | elapsed time per iteration (s): 5.42 | learning rate: 8.068E-04 | global batch size: 512 | lm loss: 3.505873E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.486 | TFLOPs: 71.37 | +[default7]: iteration 1419/ 3814 | consumed samples: 726528 | consumed tokens: 1487929344 | elapsed time per iteration (s): 5.43 | learning rate: 8.064E-04 | global batch size: 512 | lm loss: 3.513594E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.339 | TFLOPs: 71.26 | +[default7]: iteration 1420/ 3814 | consumed samples: 727040 | consumed tokens: 1488977920 | elapsed time per iteration (s): 5.43 | learning rate: 8.061E-04 | global batch size: 512 | lm loss: 3.528723E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.250 | TFLOPs: 71.19 | +[default7]: iteration 1421/ 3814 | consumed samples: 727552 | consumed tokens: 1490026496 | elapsed time per iteration (s): 5.42 | learning rate: 8.057E-04 | global batch size: 512 | lm loss: 3.513891E+00 | loss scale: 16384.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.477 | TFLOPs: 71.36 | +[default7]: iteration 1422/ 3814 | consumed samples: 728064 | consumed tokens: 1491075072 | elapsed time per iteration (s): 5.42 | learning rate: 8.054E-04 | global batch size: 512 | lm loss: 3.507744E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.395 | TFLOPs: 71.30 | +[default7]: iteration 1423/ 3814 | consumed samples: 728576 | consumed tokens: 1492123648 | elapsed time per iteration (s): 5.40 | learning rate: 8.051E-04 | global batch size: 512 | lm loss: 3.482818E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.764 | TFLOPs: 71.58 | +[default7]: iteration 1424/ 3814 | consumed samples: 729088 | consumed tokens: 1493172224 | elapsed time per iteration (s): 5.40 | learning rate: 8.047E-04 | global batch size: 512 | lm loss: 3.488361E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.773 | TFLOPs: 71.58 | +[default7]: iteration 1425/ 3814 | consumed samples: 729600 | consumed tokens: 1494220800 | elapsed time per iteration (s): 5.39 | learning rate: 8.044E-04 | global batch size: 512 | lm loss: 3.492388E+00 | loss scale: 16384.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.989 | TFLOPs: 71.75 | +[default7]: iteration 1426/ 3814 | consumed samples: 730112 | consumed tokens: 1495269376 | elapsed time per iteration (s): 5.42 | learning rate: 8.041E-04 | global batch size: 512 | lm loss: 3.501478E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.540 | TFLOPs: 71.41 | +[default7]: iteration 1427/ 3814 | consumed samples: 730624 | consumed tokens: 1496317952 | elapsed time per iteration (s): 5.40 | learning rate: 8.037E-04 | global batch size: 512 | lm loss: 3.479815E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 1428/ 3814 | consumed samples: 731136 | consumed tokens: 1497366528 | elapsed time per iteration (s): 5.43 | learning rate: 8.034E-04 | global batch size: 512 | lm loss: 3.518264E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.372 | TFLOPs: 71.28 | +[default7]: iteration 1429/ 3814 | consumed samples: 731648 | consumed tokens: 1498415104 | elapsed time per iteration (s): 5.45 | learning rate: 8.030E-04 | global batch size: 512 | lm loss: 3.507532E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.989 | TFLOPs: 70.99 | +[default7]: iteration 1430/ 3814 | consumed samples: 732160 | consumed tokens: 1499463680 | elapsed time per iteration (s): 5.42 | learning rate: 8.027E-04 | global batch size: 512 | lm loss: 3.504441E+00 | loss scale: 16384.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.546 | TFLOPs: 71.41 | +[default7]: iteration 1431/ 3814 | consumed samples: 732672 | consumed tokens: 1500512256 | elapsed time per iteration (s): 5.42 | learning rate: 8.024E-04 | global batch size: 512 | lm loss: 3.534010E+00 | loss scale: 16384.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 1432/ 3814 | consumed samples: 733184 | consumed tokens: 1501560832 | elapsed time per iteration (s): 5.42 | learning rate: 8.020E-04 | global batch size: 512 | lm loss: 3.485144E+00 | loss scale: 16384.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.393 | TFLOPs: 71.30 | +[default7]: iteration 1433/ 3814 | consumed samples: 733696 | consumed tokens: 1502609408 | elapsed time per iteration (s): 5.41 | learning rate: 8.017E-04 | global batch size: 512 | lm loss: 3.505710E+00 | loss scale: 16384.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.583 | TFLOPs: 71.44 | +[default7]: iteration 1434/ 3814 | consumed samples: 734208 | consumed tokens: 1503657984 | elapsed time per iteration (s): 5.40 | learning rate: 8.014E-04 | global batch size: 512 | lm loss: 3.513396E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.873 | TFLOPs: 71.66 | +[default7]: iteration 1435/ 3814 | consumed samples: 734720 | consumed tokens: 1504706560 | elapsed time per iteration (s): 5.42 | learning rate: 8.010E-04 | global batch size: 512 | lm loss: 3.548281E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.518 | TFLOPs: 71.39 | +[default7]: iteration 1436/ 3814 | consumed samples: 735232 | consumed tokens: 1505755136 | elapsed time per iteration (s): 5.40 | learning rate: 8.007E-04 | global batch size: 512 | lm loss: 3.506812E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.794 | TFLOPs: 71.60 | +[default7]: iteration 1437/ 3814 | consumed samples: 735744 | consumed tokens: 1506803712 | elapsed time per iteration (s): 5.40 | learning rate: 8.003E-04 | global batch size: 512 | lm loss: 3.525091E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.64 | +[default7]: iteration 1438/ 3814 | consumed samples: 736256 | consumed tokens: 1507852288 | elapsed time per iteration (s): 5.42 | learning rate: 8.000E-04 | global batch size: 512 | lm loss: 3.537363E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.546 | TFLOPs: 71.41 | +[default7]: iteration 1439/ 3814 | consumed samples: 736768 | consumed tokens: 1508900864 | elapsed time per iteration (s): 5.41 | learning rate: 7.997E-04 | global batch size: 512 | lm loss: 3.508751E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.711 | TFLOPs: 71.54 | +[default7]: iteration 1440/ 3814 | consumed samples: 737280 | consumed tokens: 1509949440 | elapsed time per iteration (s): 5.42 | learning rate: 7.993E-04 | global batch size: 512 | lm loss: 3.506059E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.506 | TFLOPs: 71.38 | +[default7]: iteration 1441/ 3814 | consumed samples: 737792 | consumed tokens: 1510998016 | elapsed time per iteration (s): 5.40 | learning rate: 7.990E-04 | global batch size: 512 | lm loss: 3.513114E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.784 | TFLOPs: 71.59 | +[default7]: iteration 1442/ 3814 | consumed samples: 738304 | consumed tokens: 1512046592 | elapsed time per iteration (s): 5.41 | learning rate: 7.986E-04 | global batch size: 512 | lm loss: 3.518331E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.578 | TFLOPs: 71.44 | +[default7]: iteration 1443/ 3814 | consumed samples: 738816 | consumed tokens: 1513095168 | elapsed time per iteration (s): 5.41 | learning rate: 7.983E-04 | global batch size: 512 | lm loss: 3.497157E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.665 | TFLOPs: 71.50 | +[default7]: iteration 1444/ 3814 | consumed samples: 739328 | consumed tokens: 1514143744 | elapsed time per iteration (s): 5.42 | learning rate: 7.979E-04 | global batch size: 512 | lm loss: 3.492473E+00 | loss scale: 16384.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.474 | TFLOPs: 71.36 | +[default7]: iteration 1445/ 3814 | consumed samples: 739840 | consumed tokens: 1515192320 | elapsed time per iteration (s): 5.42 | learning rate: 7.976E-04 | global batch size: 512 | lm loss: 3.509110E+00 | loss scale: 16384.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.525 | TFLOPs: 71.40 | +[default7]: iteration 1446/ 3814 | consumed samples: 740352 | consumed tokens: 1516240896 | elapsed time per iteration (s): 5.41 | learning rate: 7.973E-04 | global batch size: 512 | lm loss: 3.513605E+00 | loss scale: 16384.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.607 | TFLOPs: 71.46 | +[default7]: iteration 1447/ 3814 | consumed samples: 740864 | consumed tokens: 1517289472 | elapsed time per iteration (s): 5.42 | learning rate: 7.969E-04 | global batch size: 512 | lm loss: 3.507980E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 1448/ 3814 | consumed samples: 741376 | consumed tokens: 1518338048 | elapsed time per iteration (s): 5.42 | learning rate: 7.966E-04 | global batch size: 512 | lm loss: 3.502794E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.459 | TFLOPs: 71.35 | +[default7]: iteration 1449/ 3814 | consumed samples: 741888 | consumed tokens: 1519386624 | elapsed time per iteration (s): 5.45 | learning rate: 7.962E-04 | global batch size: 512 | lm loss: 3.511983E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.901 | TFLOPs: 70.92 | +[default7]: iteration 1450/ 3814 | consumed samples: 742400 | consumed tokens: 1520435200 | elapsed time per iteration (s): 5.42 | learning rate: 7.959E-04 | global batch size: 512 | lm loss: 3.497251E+00 | loss scale: 16384.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 1451/ 3814 | consumed samples: 742912 | consumed tokens: 1521483776 | elapsed time per iteration (s): 5.43 | learning rate: 7.956E-04 | global batch size: 512 | lm loss: 3.512522E+00 | loss scale: 16384.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 1452/ 3814 | consumed samples: 743424 | consumed tokens: 1522532352 | elapsed time per iteration (s): 5.42 | learning rate: 7.952E-04 | global batch size: 512 | lm loss: 3.480965E+00 | loss scale: 16384.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.391 | TFLOPs: 71.29 | +[default7]: iteration 1453/ 3814 | consumed samples: 743936 | consumed tokens: 1523580928 | elapsed time per iteration (s): 5.43 | learning rate: 7.949E-04 | global batch size: 512 | lm loss: 3.483005E+00 | loss scale: 16384.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.335 | TFLOPs: 71.25 | +[default7]: iteration 1454/ 3814 | consumed samples: 744448 | consumed tokens: 1524629504 | elapsed time per iteration (s): 5.44 | learning rate: 7.945E-04 | global batch size: 512 | lm loss: 3.548794E+00 | loss scale: 16384.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.192 | TFLOPs: 71.14 | +[default7]: iteration 1455/ 3814 | consumed samples: 744960 | consumed tokens: 1525678080 | elapsed time per iteration (s): 5.41 | learning rate: 7.942E-04 | global batch size: 512 | lm loss: 3.521508E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 1456/ 3814 | consumed samples: 745472 | consumed tokens: 1526726656 | elapsed time per iteration (s): 5.41 | learning rate: 7.938E-04 | global batch size: 512 | lm loss: 3.485301E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.567 | TFLOPs: 71.43 | +[default7]: iteration 1457/ 3814 | consumed samples: 745984 | consumed tokens: 1527775232 | elapsed time per iteration (s): 5.42 | learning rate: 7.935E-04 | global batch size: 512 | lm loss: 3.505652E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.544 | TFLOPs: 71.41 | +[default7]: iteration 1458/ 3814 | consumed samples: 746496 | consumed tokens: 1528823808 | elapsed time per iteration (s): 5.42 | learning rate: 7.932E-04 | global batch size: 512 | lm loss: 3.508307E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.416 | TFLOPs: 71.31 | +[default7]: iteration 1459/ 3814 | consumed samples: 747008 | consumed tokens: 1529872384 | elapsed time per iteration (s): 5.41 | learning rate: 7.928E-04 | global batch size: 512 | lm loss: 3.517098E+00 | loss scale: 16384.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.567 | TFLOPs: 71.43 | +[default7]: iteration 1460/ 3814 | consumed samples: 747520 | consumed tokens: 1530920960 | elapsed time per iteration (s): 5.42 | learning rate: 7.925E-04 | global batch size: 512 | lm loss: 3.530466E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.417 | TFLOPs: 71.31 | +[default7]: iteration 1461/ 3814 | consumed samples: 748032 | consumed tokens: 1531969536 | elapsed time per iteration (s): 5.42 | learning rate: 7.921E-04 | global batch size: 512 | lm loss: 3.485476E+00 | loss scale: 16384.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.481 | TFLOPs: 71.36 | +[default7]: iteration 1462/ 3814 | consumed samples: 748544 | consumed tokens: 1533018112 | elapsed time per iteration (s): 5.41 | learning rate: 7.918E-04 | global batch size: 512 | lm loss: 3.492465E+00 | loss scale: 16384.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 1463/ 3814 | consumed samples: 749056 | consumed tokens: 1534066688 | elapsed time per iteration (s): 5.42 | learning rate: 7.914E-04 | global batch size: 512 | lm loss: 3.495650E+00 | loss scale: 16384.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.391 | TFLOPs: 71.29 | +[default7]: iteration 1464/ 3814 | consumed samples: 749568 | consumed tokens: 1535115264 | elapsed time per iteration (s): 5.43 | learning rate: 7.911E-04 | global batch size: 512 | lm loss: 3.474766E+00 | loss scale: 16384.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 1465/ 3814 | consumed samples: 750080 | consumed tokens: 1536163840 | elapsed time per iteration (s): 5.44 | learning rate: 7.907E-04 | global batch size: 512 | lm loss: 3.486482E+00 | loss scale: 16384.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.135 | TFLOPs: 71.10 | +[default7]: iteration 1466/ 3814 | consumed samples: 750592 | consumed tokens: 1537212416 | elapsed time per iteration (s): 5.42 | learning rate: 7.904E-04 | global batch size: 512 | lm loss: 3.494727E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.495 | TFLOPs: 71.37 | +[default7]: iteration 1467/ 3814 | consumed samples: 751104 | consumed tokens: 1538260992 | elapsed time per iteration (s): 5.43 | learning rate: 7.900E-04 | global batch size: 512 | lm loss: 3.472371E+00 | loss scale: 16384.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.336 | TFLOPs: 71.25 | +[default7]: iteration 1468/ 3814 | consumed samples: 751616 | consumed tokens: 1539309568 | elapsed time per iteration (s): 5.42 | learning rate: 7.897E-04 | global batch size: 512 | lm loss: 3.482667E+00 | loss scale: 16384.0 | grad norm: 0.078 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 1469/ 3814 | consumed samples: 752128 | consumed tokens: 1540358144 | elapsed time per iteration (s): 5.43 | learning rate: 7.894E-04 | global batch size: 512 | lm loss: 3.481730E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 1470/ 3814 | consumed samples: 752640 | consumed tokens: 1541406720 | elapsed time per iteration (s): 5.41 | learning rate: 7.890E-04 | global batch size: 512 | lm loss: 3.527019E+00 | loss scale: 16384.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.622 | TFLOPs: 71.47 | +[default7]: iteration 1471/ 3814 | consumed samples: 753152 | consumed tokens: 1542455296 | elapsed time per iteration (s): 5.43 | learning rate: 7.887E-04 | global batch size: 512 | lm loss: 3.487858E+00 | loss scale: 16384.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.280 | TFLOPs: 71.21 | +[default7]: iteration 1472/ 3814 | consumed samples: 753664 | consumed tokens: 1543503872 | elapsed time per iteration (s): 5.42 | learning rate: 7.883E-04 | global batch size: 512 | lm loss: 3.478150E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.547 | TFLOPs: 71.41 | +[default7]: iteration 1473/ 3814 | consumed samples: 754176 | consumed tokens: 1544552448 | elapsed time per iteration (s): 5.43 | learning rate: 7.880E-04 | global batch size: 512 | lm loss: 3.458484E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 1474/ 3814 | consumed samples: 754688 | consumed tokens: 1545601024 | elapsed time per iteration (s): 5.42 | learning rate: 7.876E-04 | global batch size: 512 | lm loss: 3.491176E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.434 | TFLOPs: 71.33 | +[default7]: iteration 1475/ 3814 | consumed samples: 755200 | consumed tokens: 1546649600 | elapsed time per iteration (s): 5.40 | learning rate: 7.873E-04 | global batch size: 512 | lm loss: 3.491397E+00 | loss scale: 16384.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.739 | TFLOPs: 71.56 | +[default7]: iteration 1476/ 3814 | consumed samples: 755712 | consumed tokens: 1547698176 | elapsed time per iteration (s): 5.42 | learning rate: 7.869E-04 | global batch size: 512 | lm loss: 3.497197E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.497 | TFLOPs: 71.37 | +[default7]: iteration 1477/ 3814 | consumed samples: 756224 | consumed tokens: 1548746752 | elapsed time per iteration (s): 5.43 | learning rate: 7.866E-04 | global batch size: 512 | lm loss: 3.505113E+00 | loss scale: 16384.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.274 | TFLOPs: 71.21 | +[default7]: iteration 1478/ 3814 | consumed samples: 756736 | consumed tokens: 1549795328 | elapsed time per iteration (s): 5.40 | learning rate: 7.862E-04 | global batch size: 512 | lm loss: 3.492321E+00 | loss scale: 16384.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.850 | TFLOPs: 71.64 | +[default7]: iteration 1479/ 3814 | consumed samples: 757248 | consumed tokens: 1550843904 | elapsed time per iteration (s): 5.42 | learning rate: 7.859E-04 | global batch size: 512 | lm loss: 3.485651E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.442 | TFLOPs: 71.33 | +[default7]: iteration 1480/ 3814 | consumed samples: 757760 | consumed tokens: 1551892480 | elapsed time per iteration (s): 5.41 | learning rate: 7.855E-04 | global batch size: 512 | lm loss: 3.518059E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.691 | TFLOPs: 71.52 | +[default7]: iteration 1481/ 3814 | consumed samples: 758272 | consumed tokens: 1552941056 | elapsed time per iteration (s): 5.42 | learning rate: 7.852E-04 | global batch size: 512 | lm loss: 3.469088E+00 | loss scale: 16384.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 1482/ 3814 | consumed samples: 758784 | consumed tokens: 1553989632 | elapsed time per iteration (s): 5.41 | learning rate: 7.848E-04 | global batch size: 512 | lm loss: 3.470283E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.582 | TFLOPs: 71.44 | +[default7]: iteration 1483/ 3814 | consumed samples: 759296 | consumed tokens: 1555038208 | elapsed time per iteration (s): 5.43 | learning rate: 7.845E-04 | global batch size: 512 | lm loss: 3.483073E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 1484/ 3814 | consumed samples: 759808 | consumed tokens: 1556086784 | elapsed time per iteration (s): 5.42 | learning rate: 7.841E-04 | global batch size: 512 | lm loss: 3.491082E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.518 | TFLOPs: 71.39 | +[default7]: iteration 1485/ 3814 | consumed samples: 760320 | consumed tokens: 1557135360 | elapsed time per iteration (s): 5.42 | learning rate: 7.838E-04 | global batch size: 512 | lm loss: 3.505812E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.421 | TFLOPs: 71.32 | +[default7]: iteration 1486/ 3814 | consumed samples: 760832 | consumed tokens: 1558183936 | elapsed time per iteration (s): 5.41 | learning rate: 7.834E-04 | global batch size: 512 | lm loss: 3.484606E+00 | loss scale: 16384.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.589 | TFLOPs: 71.44 | +[default7]: iteration 1487/ 3814 | consumed samples: 761344 | consumed tokens: 1559232512 | elapsed time per iteration (s): 5.41 | learning rate: 7.831E-04 | global batch size: 512 | lm loss: 3.477688E+00 | loss scale: 16384.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.643 | TFLOPs: 71.48 | +[default7]: iteration 1488/ 3814 | consumed samples: 761856 | consumed tokens: 1560281088 | elapsed time per iteration (s): 5.41 | learning rate: 7.827E-04 | global batch size: 512 | lm loss: 3.482084E+00 | loss scale: 16384.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 1489/ 3814 | consumed samples: 762368 | consumed tokens: 1561329664 | elapsed time per iteration (s): 5.42 | learning rate: 7.824E-04 | global batch size: 512 | lm loss: 3.484177E+00 | loss scale: 16384.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.506 | TFLOPs: 71.38 | +[default7]: iteration 1490/ 3814 | consumed samples: 762880 | consumed tokens: 1562378240 | elapsed time per iteration (s): 5.40 | learning rate: 7.820E-04 | global batch size: 512 | lm loss: 3.460938E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.782 | TFLOPs: 71.59 | +[default7]: iteration 1491/ 3814 | consumed samples: 763392 | consumed tokens: 1563426816 | elapsed time per iteration (s): 5.42 | learning rate: 7.817E-04 | global batch size: 512 | lm loss: 3.487728E+00 | loss scale: 16384.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.434 | TFLOPs: 71.33 | +[default7]: iteration 1492/ 3814 | consumed samples: 763904 | consumed tokens: 1564475392 | elapsed time per iteration (s): 5.40 | learning rate: 7.813E-04 | global batch size: 512 | lm loss: 3.482950E+00 | loss scale: 16384.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.814 | TFLOPs: 71.61 | +[default7]: iteration 1493/ 3814 | consumed samples: 764416 | consumed tokens: 1565523968 | elapsed time per iteration (s): 5.42 | learning rate: 7.810E-04 | global batch size: 512 | lm loss: 3.505224E+00 | loss scale: 16384.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.500 | TFLOPs: 71.38 | +[default7]: iteration 1494/ 3814 | consumed samples: 764928 | consumed tokens: 1566572544 | elapsed time per iteration (s): 5.42 | learning rate: 7.806E-04 | global batch size: 512 | lm loss: 3.500005E+00 | loss scale: 16384.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 1495/ 3814 | consumed samples: 765440 | consumed tokens: 1567621120 | elapsed time per iteration (s): 5.42 | learning rate: 7.803E-04 | global batch size: 512 | lm loss: 3.494422E+00 | loss scale: 16384.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.505 | TFLOPs: 71.38 | +[default7]: iteration 1496/ 3814 | consumed samples: 765952 | consumed tokens: 1568669696 | elapsed time per iteration (s): 5.41 | learning rate: 7.799E-04 | global batch size: 512 | lm loss: 3.504893E+00 | loss scale: 16384.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.688 | TFLOPs: 71.52 | +[default7]: iteration 1497/ 3814 | consumed samples: 766464 | consumed tokens: 1569718272 | elapsed time per iteration (s): 5.41 | learning rate: 7.796E-04 | global batch size: 512 | lm loss: 3.525496E+00 | loss scale: 16384.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1498/ 3814 | consumed samples: 766976 | consumed tokens: 1570766848 | elapsed time per iteration (s): 5.41 | learning rate: 7.792E-04 | global batch size: 512 | lm loss: 3.506431E+00 | loss scale: 16384.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 1499/ 3814 | consumed samples: 767488 | consumed tokens: 1571815424 | elapsed time per iteration (s): 5.40 | learning rate: 7.789E-04 | global batch size: 512 | lm loss: 3.478056E+00 | loss scale: 16384.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.776 | TFLOPs: 71.59 | +[default7]: iteration 1500/ 3814 | consumed samples: 768000 | consumed tokens: 1572864000 | elapsed time per iteration (s): 5.41 | learning rate: 7.785E-04 | global batch size: 512 | lm loss: 3.480034E+00 | loss scale: 32768.0 | grad norm: 0.041 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default0]:saving checkpoint at iteration 1500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 08:35:16,155] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1500 is begin to save! +[default0]:[2023-02-16 08:35:16,158] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,158] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_01-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,309] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,309] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,310] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,311] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,372] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,373] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,375] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,376] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,430] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:16,482] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,482] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,432] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,433] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:16,488] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,489] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:16,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,594] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,546] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:16,602] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,602] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,653] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:16,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,710] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,657] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,657] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:16,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,715] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,764] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,765] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_12-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:16,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,824] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_13-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:16,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,825] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,889] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,890] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,890] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:16,944] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:16,944] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,007] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:16,945] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:16,945] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,008] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,008] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_18-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,229] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_18-model_01-model_states.pt. +[default0]:[2023-02-16 08:35:17,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,232] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,287] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,283] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,284] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,343] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_21-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,397] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,339] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,392] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,393] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,450] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_23-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,507] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,507] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,446] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,447] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_23-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,502] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,503] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,561] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,561] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,620] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,620] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,554] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,554] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_25-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,611] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,612] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,674] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,665] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,665] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_27-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,720] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,721] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,730] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,730] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_28-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,785] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,785] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,775] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,827] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:17,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,839] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:17,891] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,891] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,880] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_31-model_01-model_states.pt. +[default0]:[2023-02-16 08:35:17,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:17,949] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:18,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:18,006] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:17,931] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:17,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:17,983] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 08:35:18,066] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:18,067] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_34-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:18,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:18,127] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_35-model_00-model_states.pt... +[default1]:[2023-02-16 08:35:18,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:18,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:18,100] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:18,101] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_35-model_01-model_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:35:18,180] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:18,181] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 08:35:18,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 08:35:18,183] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_00_model_states.pt +[default0]:[2023-02-16 08:35:18,183] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 08:35:18,185] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:35:18,151] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:18,152] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 08:35:18,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 08:35:18,154] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_01_model_states.pt +[default1]:[2023-02-16 08:35:18,154] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 08:35:18,156] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:35:18,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:35:18,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,734] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,734] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,722] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,722] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,715] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,715] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,715] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,740] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,740] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,763] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,763] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,773] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,773] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,704] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,704] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,787] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,787] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,787] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,773] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,773] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,728] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,728] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,791] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,791] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,811] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,811] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,743] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,743] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,825] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,825] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,774] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,774] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,842] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,842] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,774] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,774] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,774] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,840] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,840] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,840] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,772] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,772] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,835] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,835] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,860] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,860] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,860] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,878] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,879] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,844] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,844] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,844] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,794] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,794] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default5]:[2023-02-16 08:35:18,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default4]:[2023-02-16 08:35:18,878] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,878] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,868] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,868] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,855] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,855] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,861] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,861] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,871] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,871] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,819] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,819] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,895] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,895] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,895] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,862] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,862] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,813] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,813] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,814] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,818] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,818] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,837] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,837] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,912] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,912] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,912] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,915] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,915] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,890] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,891] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,891] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,837] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,837] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,905] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,873] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,873] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,873] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,867] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,867] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,877] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,877] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,876] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,876] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,865] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,865] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,865] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default4]:[2023-02-16 08:35:18,872] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,872] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,872] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:18,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:18,904] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:18,904] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,882] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default4]:[2023-02-16 08:35:18,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:35:18,871] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:35:18,871] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,871] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,871] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,889] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,901] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,901] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,885] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,885] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,887] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,887] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default2]:[2023-02-16 08:35:18,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:35:18,901] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:35:18,901] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default5]:[2023-02-16 08:35:18,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:35:18,975] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:35:18,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default3]:[2023-02-16 08:35:18,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:35:18,916] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,916] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default0]:[2023-02-16 08:35:18,949] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:35:18,949] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:35:18,949] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default1]:[2023-02-16 08:35:18,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:35:18,921] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:35:18,921] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default6]:[2023-02-16 08:35:18,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:35:18,911] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:35:18,911] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:[2023-02-16 08:35:19,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:35:19,007] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:35:19,007] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]:time (ms) | save-checkpoint: 2853.77 +[default0]: successfully saved checkpoint at iteration 1500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default3]:[2023-02-16 08:35:18,941] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1500/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:35:18,942] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1500 is ready now! +[default7]: iteration 1501/ 3814 | consumed samples: 768512 | consumed tokens: 1573912576 | elapsed time per iteration (s): 8.29 | learning rate: 7.782E-04 | global batch size: 512 | lm loss: 3.491088E+00 | loss scale: 32768.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.780 | TFLOPs: 46.66 | +[default7]: iteration 1502/ 3814 | consumed samples: 769024 | consumed tokens: 1574961152 | elapsed time per iteration (s): 5.41 | learning rate: 7.778E-04 | global batch size: 512 | lm loss: 3.510650E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 1503/ 3814 | consumed samples: 769536 | consumed tokens: 1576009728 | elapsed time per iteration (s): 5.43 | learning rate: 7.775E-04 | global batch size: 512 | lm loss: 3.497787E+00 | loss scale: 32768.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.375 | TFLOPs: 71.28 | +[default7]: iteration 1504/ 3814 | consumed samples: 770048 | consumed tokens: 1577058304 | elapsed time per iteration (s): 5.42 | learning rate: 7.771E-04 | global batch size: 512 | lm loss: 3.501390E+00 | loss scale: 32768.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 1505/ 3814 | consumed samples: 770560 | consumed tokens: 1578106880 | elapsed time per iteration (s): 5.38 | learning rate: 7.768E-04 | global batch size: 512 | lm loss: 3.511223E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.086 | TFLOPs: 71.82 | +[default7]: iteration 1506/ 3814 | consumed samples: 771072 | consumed tokens: 1579155456 | elapsed time per iteration (s): 5.42 | learning rate: 7.764E-04 | global batch size: 512 | lm loss: 3.464809E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 1507/ 3814 | consumed samples: 771584 | consumed tokens: 1580204032 | elapsed time per iteration (s): 5.40 | learning rate: 7.761E-04 | global batch size: 512 | lm loss: 3.508780E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 1508/ 3814 | consumed samples: 772096 | consumed tokens: 1581252608 | elapsed time per iteration (s): 5.43 | learning rate: 7.757E-04 | global batch size: 512 | lm loss: 3.470318E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1509/ 3814 | consumed samples: 772608 | consumed tokens: 1582301184 | elapsed time per iteration (s): 5.40 | learning rate: 7.753E-04 | global batch size: 512 | lm loss: 3.481813E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.789 | TFLOPs: 71.60 | +[default7]: iteration 1510/ 3814 | consumed samples: 773120 | consumed tokens: 1583349760 | elapsed time per iteration (s): 5.41 | learning rate: 7.750E-04 | global batch size: 512 | lm loss: 3.463342E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.569 | TFLOPs: 71.43 | +[default7]: iteration 1511/ 3814 | consumed samples: 773632 | consumed tokens: 1584398336 | elapsed time per iteration (s): 5.42 | learning rate: 7.746E-04 | global batch size: 512 | lm loss: 3.498800E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 1512/ 3814 | consumed samples: 774144 | consumed tokens: 1585446912 | elapsed time per iteration (s): 5.41 | learning rate: 7.743E-04 | global batch size: 512 | lm loss: 3.479303E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1513/ 3814 | consumed samples: 774656 | consumed tokens: 1586495488 | elapsed time per iteration (s): 5.40 | learning rate: 7.739E-04 | global batch size: 512 | lm loss: 3.462276E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 1514/ 3814 | consumed samples: 775168 | consumed tokens: 1587544064 | elapsed time per iteration (s): 5.40 | learning rate: 7.736E-04 | global batch size: 512 | lm loss: 3.478058E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.890 | TFLOPs: 71.67 | +[default7]: iteration 1515/ 3814 | consumed samples: 775680 | consumed tokens: 1588592640 | elapsed time per iteration (s): 5.42 | learning rate: 7.732E-04 | global batch size: 512 | lm loss: 3.444435E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 1516/ 3814 | consumed samples: 776192 | consumed tokens: 1589641216 | elapsed time per iteration (s): 5.40 | learning rate: 7.729E-04 | global batch size: 512 | lm loss: 3.485500E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.789 | TFLOPs: 71.60 | +[default7]: iteration 1517/ 3814 | consumed samples: 776704 | consumed tokens: 1590689792 | elapsed time per iteration (s): 5.41 | learning rate: 7.725E-04 | global batch size: 512 | lm loss: 3.459854E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 1518/ 3814 | consumed samples: 777216 | consumed tokens: 1591738368 | elapsed time per iteration (s): 5.39 | learning rate: 7.722E-04 | global batch size: 512 | lm loss: 3.483092E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.015 | TFLOPs: 71.77 | +[default7]: iteration 1519/ 3814 | consumed samples: 777728 | consumed tokens: 1592786944 | elapsed time per iteration (s): 5.39 | learning rate: 7.718E-04 | global batch size: 512 | lm loss: 3.484661E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.072 | TFLOPs: 71.81 | +[default7]: iteration 1520/ 3814 | consumed samples: 778240 | consumed tokens: 1593835520 | elapsed time per iteration (s): 5.42 | learning rate: 7.714E-04 | global batch size: 512 | lm loss: 3.466117E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.534 | TFLOPs: 71.40 | +[default7]: iteration 1521/ 3814 | consumed samples: 778752 | consumed tokens: 1594884096 | elapsed time per iteration (s): 5.44 | learning rate: 7.711E-04 | global batch size: 512 | lm loss: 3.485837E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.137 | TFLOPs: 71.10 | +[default7]: iteration 1522/ 3814 | consumed samples: 779264 | consumed tokens: 1595932672 | elapsed time per iteration (s): 5.42 | learning rate: 7.707E-04 | global batch size: 512 | lm loss: 3.470003E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 1523/ 3814 | consumed samples: 779776 | consumed tokens: 1596981248 | elapsed time per iteration (s): 5.39 | learning rate: 7.704E-04 | global batch size: 512 | lm loss: 3.459726E+00 | loss scale: 32768.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.955 | TFLOPs: 71.72 | +[default7]: iteration 1524/ 3814 | consumed samples: 780288 | consumed tokens: 1598029824 | elapsed time per iteration (s): 5.41 | learning rate: 7.700E-04 | global batch size: 512 | lm loss: 3.466774E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.621 | TFLOPs: 71.47 | +[default7]: iteration 1525/ 3814 | consumed samples: 780800 | consumed tokens: 1599078400 | elapsed time per iteration (s): 5.41 | learning rate: 7.697E-04 | global batch size: 512 | lm loss: 3.450357E+00 | loss scale: 32768.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.574 | TFLOPs: 71.43 | +[default7]: iteration 1526/ 3814 | consumed samples: 781312 | consumed tokens: 1600126976 | elapsed time per iteration (s): 5.41 | learning rate: 7.693E-04 | global batch size: 512 | lm loss: 3.468779E+00 | loss scale: 32768.0 | grad norm: 0.080 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.593 | TFLOPs: 71.45 | +[default7]: iteration 1527/ 3814 | consumed samples: 781824 | consumed tokens: 1601175552 | elapsed time per iteration (s): 5.42 | learning rate: 7.689E-04 | global batch size: 512 | lm loss: 3.472119E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.437 | TFLOPs: 71.33 | +[default7]: iteration 1528/ 3814 | consumed samples: 782336 | consumed tokens: 1602224128 | elapsed time per iteration (s): 5.42 | learning rate: 7.686E-04 | global batch size: 512 | lm loss: 3.444421E+00 | loss scale: 32768.0 | grad norm: 0.081 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.414 | TFLOPs: 71.31 | +[default7]: iteration 1529/ 3814 | consumed samples: 782848 | consumed tokens: 1603272704 | elapsed time per iteration (s): 5.40 | learning rate: 7.682E-04 | global batch size: 512 | lm loss: 3.464156E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.792 | TFLOPs: 71.60 | +[default7]: iteration 1530/ 3814 | consumed samples: 783360 | consumed tokens: 1604321280 | elapsed time per iteration (s): 5.42 | learning rate: 7.679E-04 | global batch size: 512 | lm loss: 3.467836E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.449 | TFLOPs: 71.34 | +[default7]: iteration 1531/ 3814 | consumed samples: 783872 | consumed tokens: 1605369856 | elapsed time per iteration (s): 5.42 | learning rate: 7.675E-04 | global batch size: 512 | lm loss: 3.455665E+00 | loss scale: 32768.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 1532/ 3814 | consumed samples: 784384 | consumed tokens: 1606418432 | elapsed time per iteration (s): 5.43 | learning rate: 7.672E-04 | global batch size: 512 | lm loss: 3.439440E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.287 | TFLOPs: 71.22 | +[default7]: iteration 1533/ 3814 | consumed samples: 784896 | consumed tokens: 1607467008 | elapsed time per iteration (s): 5.41 | learning rate: 7.668E-04 | global batch size: 512 | lm loss: 3.469406E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.683 | TFLOPs: 71.52 | +[default7]: iteration 1534/ 3814 | consumed samples: 785408 | consumed tokens: 1608515584 | elapsed time per iteration (s): 5.41 | learning rate: 7.664E-04 | global batch size: 512 | lm loss: 3.487804E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 1535/ 3814 | consumed samples: 785920 | consumed tokens: 1609564160 | elapsed time per iteration (s): 5.41 | learning rate: 7.661E-04 | global batch size: 512 | lm loss: 3.480456E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.667 | TFLOPs: 71.50 | +[default7]: iteration 1536/ 3814 | consumed samples: 786432 | consumed tokens: 1610612736 | elapsed time per iteration (s): 5.42 | learning rate: 7.657E-04 | global batch size: 512 | lm loss: 3.484449E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.429 | TFLOPs: 71.32 | +[default7]: iteration 1537/ 3814 | consumed samples: 786944 | consumed tokens: 1611661312 | elapsed time per iteration (s): 5.40 | learning rate: 7.654E-04 | global batch size: 512 | lm loss: 3.476643E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.891 | TFLOPs: 71.67 | +[default7]: iteration 1538/ 3814 | consumed samples: 787456 | consumed tokens: 1612709888 | elapsed time per iteration (s): 5.40 | learning rate: 7.650E-04 | global batch size: 512 | lm loss: 3.460362E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.883 | TFLOPs: 71.67 | +[default7]: iteration 1539/ 3814 | consumed samples: 787968 | consumed tokens: 1613758464 | elapsed time per iteration (s): 5.38 | learning rate: 7.646E-04 | global batch size: 512 | lm loss: 3.460299E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.106 | TFLOPs: 71.83 | +[default7]: iteration 1540/ 3814 | consumed samples: 788480 | consumed tokens: 1614807040 | elapsed time per iteration (s): 5.42 | learning rate: 7.643E-04 | global batch size: 512 | lm loss: 3.471200E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.472 | TFLOPs: 71.36 | +[default7]: iteration 1541/ 3814 | consumed samples: 788992 | consumed tokens: 1615855616 | elapsed time per iteration (s): 5.41 | learning rate: 7.639E-04 | global batch size: 512 | lm loss: 3.465369E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.600 | TFLOPs: 71.45 | +[default7]: iteration 1542/ 3814 | consumed samples: 789504 | consumed tokens: 1616904192 | elapsed time per iteration (s): 5.41 | learning rate: 7.636E-04 | global batch size: 512 | lm loss: 3.485101E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 1543/ 3814 | consumed samples: 790016 | consumed tokens: 1617952768 | elapsed time per iteration (s): 5.40 | learning rate: 7.632E-04 | global batch size: 512 | lm loss: 3.448569E+00 | loss scale: 32768.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 1544/ 3814 | consumed samples: 790528 | consumed tokens: 1619001344 | elapsed time per iteration (s): 5.41 | learning rate: 7.628E-04 | global batch size: 512 | lm loss: 3.442732E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 1545/ 3814 | consumed samples: 791040 | consumed tokens: 1620049920 | elapsed time per iteration (s): 5.39 | learning rate: 7.625E-04 | global batch size: 512 | lm loss: 3.459210E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.911 | TFLOPs: 71.69 | +[default7]: iteration 1546/ 3814 | consumed samples: 791552 | consumed tokens: 1621098496 | elapsed time per iteration (s): 5.41 | learning rate: 7.621E-04 | global batch size: 512 | lm loss: 3.438312E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 1547/ 3814 | consumed samples: 792064 | consumed tokens: 1622147072 | elapsed time per iteration (s): 5.39 | learning rate: 7.618E-04 | global batch size: 512 | lm loss: 3.445517E+00 | loss scale: 32768.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.968 | TFLOPs: 71.73 | +[default7]: iteration 1548/ 3814 | consumed samples: 792576 | consumed tokens: 1623195648 | elapsed time per iteration (s): 5.44 | learning rate: 7.614E-04 | global batch size: 512 | lm loss: 3.447665E+00 | loss scale: 32768.0 | grad norm: 0.133 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.031 | TFLOPs: 71.02 | +[default7]: iteration 1549/ 3814 | consumed samples: 793088 | consumed tokens: 1624244224 | elapsed time per iteration (s): 5.39 | learning rate: 7.610E-04 | global batch size: 512 | lm loss: 3.472778E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.077 | TFLOPs: 71.81 | +[default7]: iteration 1550/ 3814 | consumed samples: 793600 | consumed tokens: 1625292800 | elapsed time per iteration (s): 5.40 | learning rate: 7.607E-04 | global batch size: 512 | lm loss: 3.465214E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.848 | TFLOPs: 71.64 | +[default7]: iteration 1551/ 3814 | consumed samples: 794112 | consumed tokens: 1626341376 | elapsed time per iteration (s): 5.41 | learning rate: 7.603E-04 | global batch size: 512 | lm loss: 3.445307E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.703 | TFLOPs: 71.53 | +[default7]: iteration 1552/ 3814 | consumed samples: 794624 | consumed tokens: 1627389952 | elapsed time per iteration (s): 5.41 | learning rate: 7.600E-04 | global batch size: 512 | lm loss: 3.459998E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1553/ 3814 | consumed samples: 795136 | consumed tokens: 1628438528 | elapsed time per iteration (s): 5.43 | learning rate: 7.596E-04 | global batch size: 512 | lm loss: 3.452927E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.372 | TFLOPs: 71.28 | +[default7]: iteration 1554/ 3814 | consumed samples: 795648 | consumed tokens: 1629487104 | elapsed time per iteration (s): 5.41 | learning rate: 7.592E-04 | global batch size: 512 | lm loss: 3.456195E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.628 | TFLOPs: 71.47 | +[default7]: iteration 1555/ 3814 | consumed samples: 796160 | consumed tokens: 1630535680 | elapsed time per iteration (s): 5.42 | learning rate: 7.589E-04 | global batch size: 512 | lm loss: 3.433405E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.511 | TFLOPs: 71.38 | +[default7]: iteration 1556/ 3814 | consumed samples: 796672 | consumed tokens: 1631584256 | elapsed time per iteration (s): 5.41 | learning rate: 7.585E-04 | global batch size: 512 | lm loss: 3.479325E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.724 | TFLOPs: 71.55 | +[default7]: iteration 1557/ 3814 | consumed samples: 797184 | consumed tokens: 1632632832 | elapsed time per iteration (s): 5.39 | learning rate: 7.581E-04 | global batch size: 512 | lm loss: 3.450980E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.068 | TFLOPs: 71.81 | +[default7]: iteration 1558/ 3814 | consumed samples: 797696 | consumed tokens: 1633681408 | elapsed time per iteration (s): 5.40 | learning rate: 7.578E-04 | global batch size: 512 | lm loss: 3.436432E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.900 | TFLOPs: 71.68 | +[default7]: iteration 1559/ 3814 | consumed samples: 798208 | consumed tokens: 1634729984 | elapsed time per iteration (s): 5.39 | learning rate: 7.574E-04 | global batch size: 512 | lm loss: 3.449606E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.988 | TFLOPs: 71.75 | +[default7]: iteration 1560/ 3814 | consumed samples: 798720 | consumed tokens: 1635778560 | elapsed time per iteration (s): 5.40 | learning rate: 7.571E-04 | global batch size: 512 | lm loss: 3.472919E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.859 | TFLOPs: 71.65 | +[default7]: iteration 1561/ 3814 | consumed samples: 799232 | consumed tokens: 1636827136 | elapsed time per iteration (s): 5.40 | learning rate: 7.567E-04 | global batch size: 512 | lm loss: 3.465225E+00 | loss scale: 32768.0 | grad norm: 0.079 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.850 | TFLOPs: 71.64 | +[default7]: iteration 1562/ 3814 | consumed samples: 799744 | consumed tokens: 1637875712 | elapsed time per iteration (s): 5.42 | learning rate: 7.563E-04 | global batch size: 512 | lm loss: 3.443068E+00 | loss scale: 32768.0 | grad norm: 0.083 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.423 | TFLOPs: 71.32 | +[default7]: iteration 1563/ 3814 | consumed samples: 800256 | consumed tokens: 1638924288 | elapsed time per iteration (s): 5.42 | learning rate: 7.560E-04 | global batch size: 512 | lm loss: 3.440860E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.466 | TFLOPs: 71.35 | +[default7]: iteration 1564/ 3814 | consumed samples: 800768 | consumed tokens: 1639972864 | elapsed time per iteration (s): 5.42 | learning rate: 7.556E-04 | global batch size: 512 | lm loss: 3.445798E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.410 | TFLOPs: 71.31 | +[default7]: iteration 1565/ 3814 | consumed samples: 801280 | consumed tokens: 1641021440 | elapsed time per iteration (s): 5.43 | learning rate: 7.552E-04 | global batch size: 512 | lm loss: 3.470530E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.342 | TFLOPs: 71.26 | +[default7]: iteration 1566/ 3814 | consumed samples: 801792 | consumed tokens: 1642070016 | elapsed time per iteration (s): 5.43 | learning rate: 7.549E-04 | global batch size: 512 | lm loss: 3.462632E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1567/ 3814 | consumed samples: 802304 | consumed tokens: 1643118592 | elapsed time per iteration (s): 5.39 | learning rate: 7.545E-04 | global batch size: 512 | lm loss: 3.464211E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.926 | TFLOPs: 71.70 | +[default7]: iteration 1568/ 3814 | consumed samples: 802816 | consumed tokens: 1644167168 | elapsed time per iteration (s): 5.40 | learning rate: 7.542E-04 | global batch size: 512 | lm loss: 3.437948E+00 | loss scale: 32768.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.839 | TFLOPs: 71.63 | +[default7]: iteration 1569/ 3814 | consumed samples: 803328 | consumed tokens: 1645215744 | elapsed time per iteration (s): 5.39 | learning rate: 7.538E-04 | global batch size: 512 | lm loss: 3.427154E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.911 | TFLOPs: 71.69 | +[default7]: iteration 1570/ 3814 | consumed samples: 803840 | consumed tokens: 1646264320 | elapsed time per iteration (s): 5.40 | learning rate: 7.534E-04 | global batch size: 512 | lm loss: 3.471056E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.772 | TFLOPs: 71.58 | +[default7]: iteration 1571/ 3814 | consumed samples: 804352 | consumed tokens: 1647312896 | elapsed time per iteration (s): 5.40 | learning rate: 7.531E-04 | global batch size: 512 | lm loss: 3.442178E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.764 | TFLOPs: 71.58 | +[default7]: iteration 1572/ 3814 | consumed samples: 804864 | consumed tokens: 1648361472 | elapsed time per iteration (s): 5.41 | learning rate: 7.527E-04 | global batch size: 512 | lm loss: 3.460423E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.642 | TFLOPs: 71.48 | +[default7]: iteration 1573/ 3814 | consumed samples: 805376 | consumed tokens: 1649410048 | elapsed time per iteration (s): 5.40 | learning rate: 7.523E-04 | global batch size: 512 | lm loss: 3.455964E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.877 | TFLOPs: 71.66 | +[default7]: iteration 1574/ 3814 | consumed samples: 805888 | consumed tokens: 1650458624 | elapsed time per iteration (s): 5.41 | learning rate: 7.520E-04 | global batch size: 512 | lm loss: 3.488009E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 1575/ 3814 | consumed samples: 806400 | consumed tokens: 1651507200 | elapsed time per iteration (s): 5.41 | learning rate: 7.516E-04 | global batch size: 512 | lm loss: 3.467366E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.709 | TFLOPs: 71.53 | +[default7]: iteration 1576/ 3814 | consumed samples: 806912 | consumed tokens: 1652555776 | elapsed time per iteration (s): 5.39 | learning rate: 7.512E-04 | global batch size: 512 | lm loss: 3.449958E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.942 | TFLOPs: 71.71 | +[default7]: iteration 1577/ 3814 | consumed samples: 807424 | consumed tokens: 1653604352 | elapsed time per iteration (s): 5.41 | learning rate: 7.509E-04 | global batch size: 512 | lm loss: 3.449158E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 1578/ 3814 | consumed samples: 807936 | consumed tokens: 1654652928 | elapsed time per iteration (s): 5.40 | learning rate: 7.505E-04 | global batch size: 512 | lm loss: 3.447161E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.880 | TFLOPs: 71.66 | +[default7]: iteration 1579/ 3814 | consumed samples: 808448 | consumed tokens: 1655701504 | elapsed time per iteration (s): 5.42 | learning rate: 7.501E-04 | global batch size: 512 | lm loss: 3.433626E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.507 | TFLOPs: 71.38 | +[default7]: iteration 1580/ 3814 | consumed samples: 808960 | consumed tokens: 1656750080 | elapsed time per iteration (s): 5.39 | learning rate: 7.498E-04 | global batch size: 512 | lm loss: 3.460523E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.934 | TFLOPs: 71.70 | +[default7]: iteration 1581/ 3814 | consumed samples: 809472 | consumed tokens: 1657798656 | elapsed time per iteration (s): 5.40 | learning rate: 7.494E-04 | global batch size: 512 | lm loss: 3.463632E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.900 | TFLOPs: 71.68 | +[default7]: iteration 1582/ 3814 | consumed samples: 809984 | consumed tokens: 1658847232 | elapsed time per iteration (s): 5.40 | learning rate: 7.490E-04 | global batch size: 512 | lm loss: 3.469024E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.837 | TFLOPs: 71.63 | +[default7]: iteration 1583/ 3814 | consumed samples: 810496 | consumed tokens: 1659895808 | elapsed time per iteration (s): 5.41 | learning rate: 7.487E-04 | global batch size: 512 | lm loss: 3.429122E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.639 | TFLOPs: 71.48 | +[default7]: iteration 1584/ 3814 | consumed samples: 811008 | consumed tokens: 1660944384 | elapsed time per iteration (s): 5.41 | learning rate: 7.483E-04 | global batch size: 512 | lm loss: 3.450018E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.563 | TFLOPs: 71.42 | +[default7]: iteration 1585/ 3814 | consumed samples: 811520 | consumed tokens: 1661992960 | elapsed time per iteration (s): 5.40 | learning rate: 7.479E-04 | global batch size: 512 | lm loss: 3.456136E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.793 | TFLOPs: 71.60 | +[default7]: iteration 1586/ 3814 | consumed samples: 812032 | consumed tokens: 1663041536 | elapsed time per iteration (s): 5.42 | learning rate: 7.476E-04 | global batch size: 512 | lm loss: 3.414361E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.452 | TFLOPs: 71.34 | +[default7]: iteration 1587/ 3814 | consumed samples: 812544 | consumed tokens: 1664090112 | elapsed time per iteration (s): 5.42 | learning rate: 7.472E-04 | global batch size: 512 | lm loss: 3.448466E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.516 | TFLOPs: 71.39 | +[default7]: iteration 1588/ 3814 | consumed samples: 813056 | consumed tokens: 1665138688 | elapsed time per iteration (s): 5.40 | learning rate: 7.468E-04 | global batch size: 512 | lm loss: 3.441018E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.900 | TFLOPs: 71.68 | +[default7]: iteration 1589/ 3814 | consumed samples: 813568 | consumed tokens: 1666187264 | elapsed time per iteration (s): 5.41 | learning rate: 7.465E-04 | global batch size: 512 | lm loss: 3.437906E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.664 | TFLOPs: 71.50 | +[default7]: iteration 1590/ 3814 | consumed samples: 814080 | consumed tokens: 1667235840 | elapsed time per iteration (s): 5.43 | learning rate: 7.461E-04 | global batch size: 512 | lm loss: 3.422806E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.283 | TFLOPs: 71.21 | +[default7]: iteration 1591/ 3814 | consumed samples: 814592 | consumed tokens: 1668284416 | elapsed time per iteration (s): 5.42 | learning rate: 7.457E-04 | global batch size: 512 | lm loss: 3.464263E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.506 | TFLOPs: 71.38 | +[default7]: iteration 1592/ 3814 | consumed samples: 815104 | consumed tokens: 1669332992 | elapsed time per iteration (s): 5.44 | learning rate: 7.454E-04 | global batch size: 512 | lm loss: 3.454497E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.060 | TFLOPs: 71.04 | +[default7]: iteration 1593/ 3814 | consumed samples: 815616 | consumed tokens: 1670381568 | elapsed time per iteration (s): 5.44 | learning rate: 7.450E-04 | global batch size: 512 | lm loss: 3.411741E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.138 | TFLOPs: 71.10 | +[default7]: iteration 1594/ 3814 | consumed samples: 816128 | consumed tokens: 1671430144 | elapsed time per iteration (s): 5.42 | learning rate: 7.446E-04 | global batch size: 512 | lm loss: 3.414470E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.386 | TFLOPs: 71.29 | +[default7]: iteration 1595/ 3814 | consumed samples: 816640 | consumed tokens: 1672478720 | elapsed time per iteration (s): 5.41 | learning rate: 7.443E-04 | global batch size: 512 | lm loss: 3.458073E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.561 | TFLOPs: 71.42 | +[default7]: iteration 1596/ 3814 | consumed samples: 817152 | consumed tokens: 1673527296 | elapsed time per iteration (s): 5.43 | learning rate: 7.439E-04 | global batch size: 512 | lm loss: 3.451025E+00 | loss scale: 32768.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.223 | TFLOPs: 71.17 | +[default7]: iteration 1597/ 3814 | consumed samples: 817664 | consumed tokens: 1674575872 | elapsed time per iteration (s): 5.43 | learning rate: 7.435E-04 | global batch size: 512 | lm loss: 3.438079E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.377 | TFLOPs: 71.28 | +[default7]: iteration 1598/ 3814 | consumed samples: 818176 | consumed tokens: 1675624448 | elapsed time per iteration (s): 5.42 | learning rate: 7.431E-04 | global batch size: 512 | lm loss: 3.468686E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.505 | TFLOPs: 71.38 | +[default7]: iteration 1599/ 3814 | consumed samples: 818688 | consumed tokens: 1676673024 | elapsed time per iteration (s): 5.42 | learning rate: 7.428E-04 | global batch size: 512 | lm loss: 3.438395E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.392 | TFLOPs: 71.30 | +[default7]: iteration 1600/ 3814 | consumed samples: 819200 | consumed tokens: 1677721600 | elapsed time per iteration (s): 5.42 | learning rate: 7.424E-04 | global batch size: 512 | lm loss: 3.458595E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.510 | TFLOPs: 71.38 | +[default7]: iteration 1601/ 3814 | consumed samples: 819712 | consumed tokens: 1678770176 | elapsed time per iteration (s): 5.42 | learning rate: 7.420E-04 | global batch size: 512 | lm loss: 3.406844E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 1602/ 3814 | consumed samples: 820224 | consumed tokens: 1679818752 | elapsed time per iteration (s): 5.42 | learning rate: 7.417E-04 | global batch size: 512 | lm loss: 3.419235E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 1603/ 3814 | consumed samples: 820736 | consumed tokens: 1680867328 | elapsed time per iteration (s): 5.41 | learning rate: 7.413E-04 | global batch size: 512 | lm loss: 3.451679E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.692 | TFLOPs: 71.52 | +[default7]: iteration 1604/ 3814 | consumed samples: 821248 | consumed tokens: 1681915904 | elapsed time per iteration (s): 5.41 | learning rate: 7.409E-04 | global batch size: 512 | lm loss: 3.423123E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.611 | TFLOPs: 71.46 | +[default7]: iteration 1605/ 3814 | consumed samples: 821760 | consumed tokens: 1682964480 | elapsed time per iteration (s): 5.40 | learning rate: 7.406E-04 | global batch size: 512 | lm loss: 3.437262E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.819 | TFLOPs: 71.62 | +[default7]: iteration 1606/ 3814 | consumed samples: 822272 | consumed tokens: 1684013056 | elapsed time per iteration (s): 5.42 | learning rate: 7.402E-04 | global batch size: 512 | lm loss: 3.455225E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.520 | TFLOPs: 71.39 | +[default7]: iteration 1607/ 3814 | consumed samples: 822784 | consumed tokens: 1685061632 | elapsed time per iteration (s): 5.41 | learning rate: 7.398E-04 | global batch size: 512 | lm loss: 3.474725E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.557 | TFLOPs: 71.42 | +[default7]: iteration 1608/ 3814 | consumed samples: 823296 | consumed tokens: 1686110208 | elapsed time per iteration (s): 5.43 | learning rate: 7.394E-04 | global batch size: 512 | lm loss: 3.449252E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.220 | TFLOPs: 71.16 | +[default7]: iteration 1609/ 3814 | consumed samples: 823808 | consumed tokens: 1687158784 | elapsed time per iteration (s): 5.41 | learning rate: 7.391E-04 | global batch size: 512 | lm loss: 3.435916E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 1610/ 3814 | consumed samples: 824320 | consumed tokens: 1688207360 | elapsed time per iteration (s): 5.42 | learning rate: 7.387E-04 | global batch size: 512 | lm loss: 3.459634E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.475 | TFLOPs: 71.36 | +[default7]: iteration 1611/ 3814 | consumed samples: 824832 | consumed tokens: 1689255936 | elapsed time per iteration (s): 5.40 | learning rate: 7.383E-04 | global batch size: 512 | lm loss: 3.409943E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.762 | TFLOPs: 71.57 | +[default7]: iteration 1612/ 3814 | consumed samples: 825344 | consumed tokens: 1690304512 | elapsed time per iteration (s): 5.41 | learning rate: 7.380E-04 | global batch size: 512 | lm loss: 3.446491E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.643 | TFLOPs: 71.48 | +[default7]: iteration 1613/ 3814 | consumed samples: 825856 | consumed tokens: 1691353088 | elapsed time per iteration (s): 5.43 | learning rate: 7.376E-04 | global batch size: 512 | lm loss: 3.429393E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.292 | TFLOPs: 71.22 | +[default7]: iteration 1614/ 3814 | consumed samples: 826368 | consumed tokens: 1692401664 | elapsed time per iteration (s): 5.45 | learning rate: 7.372E-04 | global batch size: 512 | lm loss: 3.432268E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.965 | TFLOPs: 70.97 | +[default7]: iteration 1615/ 3814 | consumed samples: 826880 | consumed tokens: 1693450240 | elapsed time per iteration (s): 5.39 | learning rate: 7.368E-04 | global batch size: 512 | lm loss: 3.447204E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.926 | TFLOPs: 71.70 | +[default7]: iteration 1616/ 3814 | consumed samples: 827392 | consumed tokens: 1694498816 | elapsed time per iteration (s): 5.43 | learning rate: 7.365E-04 | global batch size: 512 | lm loss: 3.436651E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.376 | TFLOPs: 71.28 | +[default7]: iteration 1617/ 3814 | consumed samples: 827904 | consumed tokens: 1695547392 | elapsed time per iteration (s): 5.42 | learning rate: 7.361E-04 | global batch size: 512 | lm loss: 3.430419E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.512 | TFLOPs: 71.39 | +[default7]: iteration 1618/ 3814 | consumed samples: 828416 | consumed tokens: 1696595968 | elapsed time per iteration (s): 5.39 | learning rate: 7.357E-04 | global batch size: 512 | lm loss: 3.427955E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.919 | TFLOPs: 71.69 | +[default7]: iteration 1619/ 3814 | consumed samples: 828928 | consumed tokens: 1697644544 | elapsed time per iteration (s): 5.42 | learning rate: 7.354E-04 | global batch size: 512 | lm loss: 3.438228E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.511 | TFLOPs: 71.39 | +[default7]: iteration 1620/ 3814 | consumed samples: 829440 | consumed tokens: 1698693120 | elapsed time per iteration (s): 5.43 | learning rate: 7.350E-04 | global batch size: 512 | lm loss: 3.411883E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.264 | TFLOPs: 71.20 | +[default7]: iteration 1621/ 3814 | consumed samples: 829952 | consumed tokens: 1699741696 | elapsed time per iteration (s): 5.42 | learning rate: 7.346E-04 | global batch size: 512 | lm loss: 3.426456E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.392 | TFLOPs: 71.30 | +[default7]: iteration 1622/ 3814 | consumed samples: 830464 | consumed tokens: 1700790272 | elapsed time per iteration (s): 5.42 | learning rate: 7.342E-04 | global batch size: 512 | lm loss: 3.424949E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.402 | TFLOPs: 71.30 | +[default7]: iteration 1623/ 3814 | consumed samples: 830976 | consumed tokens: 1701838848 | elapsed time per iteration (s): 5.41 | learning rate: 7.339E-04 | global batch size: 512 | lm loss: 3.434881E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.689 | TFLOPs: 71.52 | +[default7]: iteration 1624/ 3814 | consumed samples: 831488 | consumed tokens: 1702887424 | elapsed time per iteration (s): 5.41 | learning rate: 7.335E-04 | global batch size: 512 | lm loss: 3.438303E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.657 | TFLOPs: 71.50 | +[default7]: iteration 1625/ 3814 | consumed samples: 832000 | consumed tokens: 1703936000 | elapsed time per iteration (s): 5.40 | learning rate: 7.331E-04 | global batch size: 512 | lm loss: 3.390910E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.729 | TFLOPs: 71.55 | +[default7]: iteration 1626/ 3814 | consumed samples: 832512 | consumed tokens: 1704984576 | elapsed time per iteration (s): 5.41 | learning rate: 7.327E-04 | global batch size: 512 | lm loss: 3.419661E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 1627/ 3814 | consumed samples: 833024 | consumed tokens: 1706033152 | elapsed time per iteration (s): 5.42 | learning rate: 7.324E-04 | global batch size: 512 | lm loss: 3.408798E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.429 | TFLOPs: 71.32 | +[default7]: iteration 1628/ 3814 | consumed samples: 833536 | consumed tokens: 1707081728 | elapsed time per iteration (s): 5.43 | learning rate: 7.320E-04 | global batch size: 512 | lm loss: 3.434549E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.326 | TFLOPs: 71.25 | +[default7]: iteration 1629/ 3814 | consumed samples: 834048 | consumed tokens: 1708130304 | elapsed time per iteration (s): 5.42 | learning rate: 7.316E-04 | global batch size: 512 | lm loss: 3.453419E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.472 | TFLOPs: 71.36 | +[default7]: iteration 1630/ 3814 | consumed samples: 834560 | consumed tokens: 1709178880 | elapsed time per iteration (s): 5.44 | learning rate: 7.312E-04 | global batch size: 512 | lm loss: 3.424034E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.169 | TFLOPs: 71.13 | +[default7]: iteration 1631/ 3814 | consumed samples: 835072 | consumed tokens: 1710227456 | elapsed time per iteration (s): 5.42 | learning rate: 7.309E-04 | global batch size: 512 | lm loss: 3.440773E+00 | loss scale: 32768.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.505 | TFLOPs: 71.38 | +[default7]: iteration 1632/ 3814 | consumed samples: 835584 | consumed tokens: 1711276032 | elapsed time per iteration (s): 5.41 | learning rate: 7.305E-04 | global batch size: 512 | lm loss: 3.433632E+00 | loss scale: 32768.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1633/ 3814 | consumed samples: 836096 | consumed tokens: 1712324608 | elapsed time per iteration (s): 5.40 | learning rate: 7.301E-04 | global batch size: 512 | lm loss: 3.392796E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.774 | TFLOPs: 71.58 | +[default7]: iteration 1634/ 3814 | consumed samples: 836608 | consumed tokens: 1713373184 | elapsed time per iteration (s): 5.45 | learning rate: 7.297E-04 | global batch size: 512 | lm loss: 3.398342E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.942 | TFLOPs: 70.96 | +[default7]: iteration 1635/ 3814 | consumed samples: 837120 | consumed tokens: 1714421760 | elapsed time per iteration (s): 5.43 | learning rate: 7.294E-04 | global batch size: 512 | lm loss: 3.416049E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.375 | TFLOPs: 71.28 | +[default7]: iteration 1636/ 3814 | consumed samples: 837632 | consumed tokens: 1715470336 | elapsed time per iteration (s): 5.42 | learning rate: 7.290E-04 | global batch size: 512 | lm loss: 3.423590E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.539 | TFLOPs: 71.41 | +[default7]: iteration 1637/ 3814 | consumed samples: 838144 | consumed tokens: 1716518912 | elapsed time per iteration (s): 5.43 | learning rate: 7.286E-04 | global batch size: 512 | lm loss: 3.426457E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.284 | TFLOPs: 71.21 | +[default7]: iteration 1638/ 3814 | consumed samples: 838656 | consumed tokens: 1717567488 | elapsed time per iteration (s): 5.41 | learning rate: 7.282E-04 | global batch size: 512 | lm loss: 3.435411E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.724 | TFLOPs: 71.55 | +[default7]: iteration 1639/ 3814 | consumed samples: 839168 | consumed tokens: 1718616064 | elapsed time per iteration (s): 5.42 | learning rate: 7.279E-04 | global batch size: 512 | lm loss: 3.439505E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.401 | TFLOPs: 71.30 | +[default7]: iteration 1640/ 3814 | consumed samples: 839680 | consumed tokens: 1719664640 | elapsed time per iteration (s): 5.43 | learning rate: 7.275E-04 | global batch size: 512 | lm loss: 3.426690E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.346 | TFLOPs: 71.26 | +[default7]: iteration 1641/ 3814 | consumed samples: 840192 | consumed tokens: 1720713216 | elapsed time per iteration (s): 5.41 | learning rate: 7.271E-04 | global batch size: 512 | lm loss: 3.439543E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.660 | TFLOPs: 71.50 | +[default7]: iteration 1642/ 3814 | consumed samples: 840704 | consumed tokens: 1721761792 | elapsed time per iteration (s): 5.43 | learning rate: 7.267E-04 | global batch size: 512 | lm loss: 3.367818E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.355 | TFLOPs: 71.27 | +[default7]: iteration 1643/ 3814 | consumed samples: 841216 | consumed tokens: 1722810368 | elapsed time per iteration (s): 5.42 | learning rate: 7.264E-04 | global batch size: 512 | lm loss: 3.414340E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.536 | TFLOPs: 71.40 | +[default7]: iteration 1644/ 3814 | consumed samples: 841728 | consumed tokens: 1723858944 | elapsed time per iteration (s): 5.43 | learning rate: 7.260E-04 | global batch size: 512 | lm loss: 3.424471E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 1645/ 3814 | consumed samples: 842240 | consumed tokens: 1724907520 | elapsed time per iteration (s): 5.42 | learning rate: 7.256E-04 | global batch size: 512 | lm loss: 3.401148E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.480 | TFLOPs: 71.36 | +[default7]: iteration 1646/ 3814 | consumed samples: 842752 | consumed tokens: 1725956096 | elapsed time per iteration (s): 5.41 | learning rate: 7.252E-04 | global batch size: 512 | lm loss: 3.426360E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1647/ 3814 | consumed samples: 843264 | consumed tokens: 1727004672 | elapsed time per iteration (s): 5.41 | learning rate: 7.249E-04 | global batch size: 512 | lm loss: 3.409995E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.605 | TFLOPs: 71.46 | +[default7]: iteration 1648/ 3814 | consumed samples: 843776 | consumed tokens: 1728053248 | elapsed time per iteration (s): 5.41 | learning rate: 7.245E-04 | global batch size: 512 | lm loss: 3.392153E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.662 | TFLOPs: 71.50 | +[default7]: iteration 1649/ 3814 | consumed samples: 844288 | consumed tokens: 1729101824 | elapsed time per iteration (s): 5.41 | learning rate: 7.241E-04 | global batch size: 512 | lm loss: 3.443994E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1650/ 3814 | consumed samples: 844800 | consumed tokens: 1730150400 | elapsed time per iteration (s): 5.40 | learning rate: 7.237E-04 | global batch size: 512 | lm loss: 3.400942E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.902 | TFLOPs: 71.68 | +[default7]: iteration 1651/ 3814 | consumed samples: 845312 | consumed tokens: 1731198976 | elapsed time per iteration (s): 5.40 | learning rate: 7.234E-04 | global batch size: 512 | lm loss: 3.414492E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.752 | TFLOPs: 71.57 | +[default7]: iteration 1652/ 3814 | consumed samples: 845824 | consumed tokens: 1732247552 | elapsed time per iteration (s): 5.40 | learning rate: 7.230E-04 | global batch size: 512 | lm loss: 3.403729E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.744 | TFLOPs: 71.56 | +[default7]: iteration 1653/ 3814 | consumed samples: 846336 | consumed tokens: 1733296128 | elapsed time per iteration (s): 5.41 | learning rate: 7.226E-04 | global batch size: 512 | lm loss: 3.431161E+00 | loss scale: 32768.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.624 | TFLOPs: 71.47 | +[default7]: iteration 1654/ 3814 | consumed samples: 846848 | consumed tokens: 1734344704 | elapsed time per iteration (s): 5.40 | learning rate: 7.222E-04 | global batch size: 512 | lm loss: 3.397363E+00 | loss scale: 32768.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.774 | TFLOPs: 71.58 | +[default7]: iteration 1655/ 3814 | consumed samples: 847360 | consumed tokens: 1735393280 | elapsed time per iteration (s): 5.39 | learning rate: 7.218E-04 | global batch size: 512 | lm loss: 3.401067E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.980 | TFLOPs: 71.74 | +[default7]: iteration 1656/ 3814 | consumed samples: 847872 | consumed tokens: 1736441856 | elapsed time per iteration (s): 5.39 | learning rate: 7.215E-04 | global batch size: 512 | lm loss: 3.410439E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.050 | TFLOPs: 71.79 | +[default7]: iteration 1657/ 3814 | consumed samples: 848384 | consumed tokens: 1737490432 | elapsed time per iteration (s): 5.42 | learning rate: 7.211E-04 | global batch size: 512 | lm loss: 3.415597E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.398 | TFLOPs: 71.30 | +[default7]: iteration 1658/ 3814 | consumed samples: 848896 | consumed tokens: 1738539008 | elapsed time per iteration (s): 5.44 | learning rate: 7.207E-04 | global batch size: 512 | lm loss: 3.434215E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.083 | TFLOPs: 71.06 | +[default7]: iteration 1659/ 3814 | consumed samples: 849408 | consumed tokens: 1739587584 | elapsed time per iteration (s): 5.43 | learning rate: 7.203E-04 | global batch size: 512 | lm loss: 3.406137E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.297 | TFLOPs: 71.22 | +[default7]: iteration 1660/ 3814 | consumed samples: 849920 | consumed tokens: 1740636160 | elapsed time per iteration (s): 5.41 | learning rate: 7.199E-04 | global batch size: 512 | lm loss: 3.432964E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 1661/ 3814 | consumed samples: 850432 | consumed tokens: 1741684736 | elapsed time per iteration (s): 5.39 | learning rate: 7.196E-04 | global batch size: 512 | lm loss: 3.443347E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.929 | TFLOPs: 71.70 | +[default7]: iteration 1662/ 3814 | consumed samples: 850944 | consumed tokens: 1742733312 | elapsed time per iteration (s): 5.41 | learning rate: 7.192E-04 | global batch size: 512 | lm loss: 3.409019E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 1663/ 3814 | consumed samples: 851456 | consumed tokens: 1743781888 | elapsed time per iteration (s): 5.40 | learning rate: 7.188E-04 | global batch size: 512 | lm loss: 3.429029E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.839 | TFLOPs: 71.63 | +[default7]: iteration 1664/ 3814 | consumed samples: 851968 | consumed tokens: 1744830464 | elapsed time per iteration (s): 5.40 | learning rate: 7.184E-04 | global batch size: 512 | lm loss: 3.406017E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.872 | TFLOPs: 71.66 | +[default7]: iteration 1665/ 3814 | consumed samples: 852480 | consumed tokens: 1745879040 | elapsed time per iteration (s): 5.41 | learning rate: 7.181E-04 | global batch size: 512 | lm loss: 3.429882E+00 | loss scale: 32768.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.658 | TFLOPs: 71.50 | +[default7]: iteration 1666/ 3814 | consumed samples: 852992 | consumed tokens: 1746927616 | elapsed time per iteration (s): 5.42 | learning rate: 7.177E-04 | global batch size: 512 | lm loss: 3.417550E+00 | loss scale: 32768.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 1667/ 3814 | consumed samples: 853504 | consumed tokens: 1747976192 | elapsed time per iteration (s): 5.42 | learning rate: 7.173E-04 | global batch size: 512 | lm loss: 3.415664E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.443 | TFLOPs: 71.33 | +[default7]: iteration 1668/ 3814 | consumed samples: 854016 | consumed tokens: 1749024768 | elapsed time per iteration (s): 5.41 | learning rate: 7.169E-04 | global batch size: 512 | lm loss: 3.370158E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.594 | TFLOPs: 71.45 | +[default7]: iteration 1669/ 3814 | consumed samples: 854528 | consumed tokens: 1750073344 | elapsed time per iteration (s): 5.39 | learning rate: 7.165E-04 | global batch size: 512 | lm loss: 3.424817E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.929 | TFLOPs: 71.70 | +[default7]: iteration 1670/ 3814 | consumed samples: 855040 | consumed tokens: 1751121920 | elapsed time per iteration (s): 5.41 | learning rate: 7.162E-04 | global batch size: 512 | lm loss: 3.425732E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 1671/ 3814 | consumed samples: 855552 | consumed tokens: 1752170496 | elapsed time per iteration (s): 5.40 | learning rate: 7.158E-04 | global batch size: 512 | lm loss: 3.396213E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.741 | TFLOPs: 71.56 | +[default7]: iteration 1672/ 3814 | consumed samples: 856064 | consumed tokens: 1753219072 | elapsed time per iteration (s): 5.40 | learning rate: 7.154E-04 | global batch size: 512 | lm loss: 3.428657E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.837 | TFLOPs: 71.63 | +[default7]: iteration 1673/ 3814 | consumed samples: 856576 | consumed tokens: 1754267648 | elapsed time per iteration (s): 5.40 | learning rate: 7.150E-04 | global batch size: 512 | lm loss: 3.371302E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.803 | TFLOPs: 71.61 | +[default7]: iteration 1674/ 3814 | consumed samples: 857088 | consumed tokens: 1755316224 | elapsed time per iteration (s): 5.41 | learning rate: 7.146E-04 | global batch size: 512 | lm loss: 3.414267E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.721 | TFLOPs: 71.54 | +[default7]: iteration 1675/ 3814 | consumed samples: 857600 | consumed tokens: 1756364800 | elapsed time per iteration (s): 5.42 | learning rate: 7.143E-04 | global batch size: 512 | lm loss: 3.403092E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.465 | TFLOPs: 71.35 | +[default7]: iteration 1676/ 3814 | consumed samples: 858112 | consumed tokens: 1757413376 | elapsed time per iteration (s): 5.42 | learning rate: 7.139E-04 | global batch size: 512 | lm loss: 3.381449E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.472 | TFLOPs: 71.36 | +[default7]: iteration 1677/ 3814 | consumed samples: 858624 | consumed tokens: 1758461952 | elapsed time per iteration (s): 5.39 | learning rate: 7.135E-04 | global batch size: 512 | lm loss: 3.440960E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.980 | TFLOPs: 71.74 | +[default7]: iteration 1678/ 3814 | consumed samples: 859136 | consumed tokens: 1759510528 | elapsed time per iteration (s): 5.43 | learning rate: 7.131E-04 | global batch size: 512 | lm loss: 3.404563E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1679/ 3814 | consumed samples: 859648 | consumed tokens: 1760559104 | elapsed time per iteration (s): 5.41 | learning rate: 7.127E-04 | global batch size: 512 | lm loss: 3.431947E+00 | loss scale: 32768.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default7]: iteration 1680/ 3814 | consumed samples: 860160 | consumed tokens: 1761607680 | elapsed time per iteration (s): 5.41 | learning rate: 7.123E-04 | global batch size: 512 | lm loss: 3.381332E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.641 | TFLOPs: 71.48 | +[default7]: iteration 1681/ 3814 | consumed samples: 860672 | consumed tokens: 1762656256 | elapsed time per iteration (s): 5.42 | learning rate: 7.120E-04 | global batch size: 512 | lm loss: 3.413723E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.428 | TFLOPs: 71.32 | +[default7]: iteration 1682/ 3814 | consumed samples: 861184 | consumed tokens: 1763704832 | elapsed time per iteration (s): 5.39 | learning rate: 7.116E-04 | global batch size: 512 | lm loss: 3.421650E+00 | loss scale: 32768.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.948 | TFLOPs: 71.72 | +[default7]: iteration 1683/ 3814 | consumed samples: 861696 | consumed tokens: 1764753408 | elapsed time per iteration (s): 5.41 | learning rate: 7.112E-04 | global batch size: 512 | lm loss: 3.416826E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.692 | TFLOPs: 71.52 | +[default7]: iteration 1684/ 3814 | consumed samples: 862208 | consumed tokens: 1765801984 | elapsed time per iteration (s): 5.40 | learning rate: 7.108E-04 | global batch size: 512 | lm loss: 3.401603E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.800 | TFLOPs: 71.60 | +[default7]: iteration 1685/ 3814 | consumed samples: 862720 | consumed tokens: 1766850560 | elapsed time per iteration (s): 5.42 | learning rate: 7.104E-04 | global batch size: 512 | lm loss: 3.432557E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 1686/ 3814 | consumed samples: 863232 | consumed tokens: 1767899136 | elapsed time per iteration (s): 5.42 | learning rate: 7.101E-04 | global batch size: 512 | lm loss: 3.410134E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.465 | TFLOPs: 71.35 | +[default7]: iteration 1687/ 3814 | consumed samples: 863744 | consumed tokens: 1768947712 | elapsed time per iteration (s): 5.43 | learning rate: 7.097E-04 | global batch size: 512 | lm loss: 3.416980E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.248 | TFLOPs: 71.19 | +[default7]: iteration 1688/ 3814 | consumed samples: 864256 | consumed tokens: 1769996288 | elapsed time per iteration (s): 5.43 | learning rate: 7.093E-04 | global batch size: 512 | lm loss: 3.394848E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.366 | TFLOPs: 71.28 | +[default7]: iteration 1689/ 3814 | consumed samples: 864768 | consumed tokens: 1771044864 | elapsed time per iteration (s): 5.42 | learning rate: 7.089E-04 | global batch size: 512 | lm loss: 3.389801E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.385 | TFLOPs: 71.29 | +[default7]: iteration 1690/ 3814 | consumed samples: 865280 | consumed tokens: 1772093440 | elapsed time per iteration (s): 5.41 | learning rate: 7.085E-04 | global batch size: 512 | lm loss: 3.418699E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.696 | TFLOPs: 71.52 | +[default7]: iteration 1691/ 3814 | consumed samples: 865792 | consumed tokens: 1773142016 | elapsed time per iteration (s): 5.41 | learning rate: 7.081E-04 | global batch size: 512 | lm loss: 3.394892E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.643 | TFLOPs: 71.49 | +[default7]: iteration 1692/ 3814 | consumed samples: 866304 | consumed tokens: 1774190592 | elapsed time per iteration (s): 5.42 | learning rate: 7.078E-04 | global batch size: 512 | lm loss: 3.427547E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.528 | TFLOPs: 71.40 | +[default7]: iteration 1693/ 3814 | consumed samples: 866816 | consumed tokens: 1775239168 | elapsed time per iteration (s): 5.41 | learning rate: 7.074E-04 | global batch size: 512 | lm loss: 3.403563E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 1694/ 3814 | consumed samples: 867328 | consumed tokens: 1776287744 | elapsed time per iteration (s): 5.41 | learning rate: 7.070E-04 | global batch size: 512 | lm loss: 3.414107E+00 | loss scale: 32768.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.617 | TFLOPs: 71.47 | +[default7]: iteration 1695/ 3814 | consumed samples: 867840 | consumed tokens: 1777336320 | elapsed time per iteration (s): 5.41 | learning rate: 7.066E-04 | global batch size: 512 | lm loss: 3.378000E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 1696/ 3814 | consumed samples: 868352 | consumed tokens: 1778384896 | elapsed time per iteration (s): 5.41 | learning rate: 7.062E-04 | global batch size: 512 | lm loss: 3.407772E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 1697/ 3814 | consumed samples: 868864 | consumed tokens: 1779433472 | elapsed time per iteration (s): 5.42 | learning rate: 7.058E-04 | global batch size: 512 | lm loss: 3.418612E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.544 | TFLOPs: 71.41 | +[default7]: iteration 1698/ 3814 | consumed samples: 869376 | consumed tokens: 1780482048 | elapsed time per iteration (s): 5.42 | learning rate: 7.055E-04 | global batch size: 512 | lm loss: 3.389110E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.380 | TFLOPs: 71.29 | +[default7]: iteration 1699/ 3814 | consumed samples: 869888 | consumed tokens: 1781530624 | elapsed time per iteration (s): 5.40 | learning rate: 7.051E-04 | global batch size: 512 | lm loss: 3.385216E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.879 | TFLOPs: 71.66 | +[default7]: iteration 1700/ 3814 | consumed samples: 870400 | consumed tokens: 1782579200 | elapsed time per iteration (s): 5.41 | learning rate: 7.047E-04 | global batch size: 512 | lm loss: 3.413583E+00 | loss scale: 32768.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.669 | TFLOPs: 71.50 | +[default7]: iteration 1701/ 3814 | consumed samples: 870912 | consumed tokens: 1783627776 | elapsed time per iteration (s): 5.42 | learning rate: 7.043E-04 | global batch size: 512 | lm loss: 3.400917E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 1702/ 3814 | consumed samples: 871424 | consumed tokens: 1784676352 | elapsed time per iteration (s): 5.41 | learning rate: 7.039E-04 | global batch size: 512 | lm loss: 3.428006E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 1703/ 3814 | consumed samples: 871936 | consumed tokens: 1785724928 | elapsed time per iteration (s): 5.41 | learning rate: 7.035E-04 | global batch size: 512 | lm loss: 3.395019E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 1704/ 3814 | consumed samples: 872448 | consumed tokens: 1786773504 | elapsed time per iteration (s): 5.40 | learning rate: 7.032E-04 | global batch size: 512 | lm loss: 3.396777E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.805 | TFLOPs: 71.61 | +[default7]: iteration 1705/ 3814 | consumed samples: 872960 | consumed tokens: 1787822080 | elapsed time per iteration (s): 5.41 | learning rate: 7.028E-04 | global batch size: 512 | lm loss: 3.380889E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 1706/ 3814 | consumed samples: 873472 | consumed tokens: 1788870656 | elapsed time per iteration (s): 5.41 | learning rate: 7.024E-04 | global batch size: 512 | lm loss: 3.415592E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.669 | TFLOPs: 71.50 | +[default7]: iteration 1707/ 3814 | consumed samples: 873984 | consumed tokens: 1789919232 | elapsed time per iteration (s): 5.42 | learning rate: 7.020E-04 | global batch size: 512 | lm loss: 3.392390E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 1708/ 3814 | consumed samples: 874496 | consumed tokens: 1790967808 | elapsed time per iteration (s): 5.44 | learning rate: 7.016E-04 | global batch size: 512 | lm loss: 3.395941E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.082 | TFLOPs: 71.06 | +[default7]: iteration 1709/ 3814 | consumed samples: 875008 | consumed tokens: 1792016384 | elapsed time per iteration (s): 5.44 | learning rate: 7.012E-04 | global batch size: 512 | lm loss: 3.404591E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.141 | TFLOPs: 71.11 | +[default7]: iteration 1710/ 3814 | consumed samples: 875520 | consumed tokens: 1793064960 | elapsed time per iteration (s): 5.43 | learning rate: 7.008E-04 | global batch size: 512 | lm loss: 3.396600E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.274 | TFLOPs: 71.21 | +[default7]: iteration 1711/ 3814 | consumed samples: 876032 | consumed tokens: 1794113536 | elapsed time per iteration (s): 5.42 | learning rate: 7.005E-04 | global batch size: 512 | lm loss: 3.406112E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.530 | TFLOPs: 71.40 | +[default7]: iteration 1712/ 3814 | consumed samples: 876544 | consumed tokens: 1795162112 | elapsed time per iteration (s): 5.42 | learning rate: 7.001E-04 | global batch size: 512 | lm loss: 3.383884E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.386 | TFLOPs: 71.29 | +[default7]: iteration 1713/ 3814 | consumed samples: 877056 | consumed tokens: 1796210688 | elapsed time per iteration (s): 5.42 | learning rate: 6.997E-04 | global batch size: 512 | lm loss: 3.389425E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.394 | TFLOPs: 71.30 | +[default7]: iteration 1714/ 3814 | consumed samples: 877568 | consumed tokens: 1797259264 | elapsed time per iteration (s): 5.40 | learning rate: 6.993E-04 | global batch size: 512 | lm loss: 3.403250E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.753 | TFLOPs: 71.57 | +[default7]: iteration 1715/ 3814 | consumed samples: 878080 | consumed tokens: 1798307840 | elapsed time per iteration (s): 5.42 | learning rate: 6.989E-04 | global batch size: 512 | lm loss: 3.378543E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.397 | TFLOPs: 71.30 | +[default7]: iteration 1716/ 3814 | consumed samples: 878592 | consumed tokens: 1799356416 | elapsed time per iteration (s): 5.42 | learning rate: 6.985E-04 | global batch size: 512 | lm loss: 3.420966E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.545 | TFLOPs: 71.41 | +[default7]: iteration 1717/ 3814 | consumed samples: 879104 | consumed tokens: 1800404992 | elapsed time per iteration (s): 5.40 | learning rate: 6.981E-04 | global batch size: 512 | lm loss: 3.389385E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.739 | TFLOPs: 71.56 | +[default7]: iteration 1718/ 3814 | consumed samples: 879616 | consumed tokens: 1801453568 | elapsed time per iteration (s): 5.41 | learning rate: 6.978E-04 | global batch size: 512 | lm loss: 3.405561E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.724 | TFLOPs: 71.55 | +[default7]: iteration 1719/ 3814 | consumed samples: 880128 | consumed tokens: 1802502144 | elapsed time per iteration (s): 5.41 | learning rate: 6.974E-04 | global batch size: 512 | lm loss: 3.394162E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.712 | TFLOPs: 71.54 | +[default7]: iteration 1720/ 3814 | consumed samples: 880640 | consumed tokens: 1803550720 | elapsed time per iteration (s): 5.41 | learning rate: 6.970E-04 | global batch size: 512 | lm loss: 3.389784E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 1721/ 3814 | consumed samples: 881152 | consumed tokens: 1804599296 | elapsed time per iteration (s): 5.42 | learning rate: 6.966E-04 | global batch size: 512 | lm loss: 3.416875E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 1722/ 3814 | consumed samples: 881664 | consumed tokens: 1805647872 | elapsed time per iteration (s): 5.41 | learning rate: 6.962E-04 | global batch size: 512 | lm loss: 3.401930E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.647 | TFLOPs: 71.49 | +[default7]: iteration 1723/ 3814 | consumed samples: 882176 | consumed tokens: 1806696448 | elapsed time per iteration (s): 5.44 | learning rate: 6.958E-04 | global batch size: 512 | lm loss: 3.371780E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.079 | TFLOPs: 71.06 | +[default7]: iteration 1724/ 3814 | consumed samples: 882688 | consumed tokens: 1807745024 | elapsed time per iteration (s): 5.43 | learning rate: 6.954E-04 | global batch size: 512 | lm loss: 3.391631E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.232 | TFLOPs: 71.17 | +[default7]: iteration 1725/ 3814 | consumed samples: 883200 | consumed tokens: 1808793600 | elapsed time per iteration (s): 5.42 | learning rate: 6.951E-04 | global batch size: 512 | lm loss: 3.393583E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.382 | TFLOPs: 71.29 | +[default7]: iteration 1726/ 3814 | consumed samples: 883712 | consumed tokens: 1809842176 | elapsed time per iteration (s): 5.42 | learning rate: 6.947E-04 | global batch size: 512 | lm loss: 3.406325E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.547 | TFLOPs: 71.41 | +[default7]: iteration 1727/ 3814 | consumed samples: 884224 | consumed tokens: 1810890752 | elapsed time per iteration (s): 5.41 | learning rate: 6.943E-04 | global batch size: 512 | lm loss: 3.417681E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.665 | TFLOPs: 71.50 | +[default7]: iteration 1728/ 3814 | consumed samples: 884736 | consumed tokens: 1811939328 | elapsed time per iteration (s): 5.45 | learning rate: 6.939E-04 | global batch size: 512 | lm loss: 3.386190E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.913 | TFLOPs: 70.93 | +[default7]: iteration 1729/ 3814 | consumed samples: 885248 | consumed tokens: 1812987904 | elapsed time per iteration (s): 5.42 | learning rate: 6.935E-04 | global batch size: 512 | lm loss: 3.432421E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 1730/ 3814 | consumed samples: 885760 | consumed tokens: 1814036480 | elapsed time per iteration (s): 5.39 | learning rate: 6.931E-04 | global batch size: 512 | lm loss: 3.392791E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.907 | TFLOPs: 71.68 | +[default7]: iteration 1731/ 3814 | consumed samples: 886272 | consumed tokens: 1815085056 | elapsed time per iteration (s): 5.40 | learning rate: 6.927E-04 | global batch size: 512 | lm loss: 3.381849E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.822 | TFLOPs: 71.62 | +[default7]: iteration 1732/ 3814 | consumed samples: 886784 | consumed tokens: 1816133632 | elapsed time per iteration (s): 5.39 | learning rate: 6.923E-04 | global batch size: 512 | lm loss: 3.412416E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.940 | TFLOPs: 71.71 | +[default7]: iteration 1733/ 3814 | consumed samples: 887296 | consumed tokens: 1817182208 | elapsed time per iteration (s): 5.40 | learning rate: 6.920E-04 | global batch size: 512 | lm loss: 3.374401E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.757 | TFLOPs: 71.57 | +[default7]: iteration 1734/ 3814 | consumed samples: 887808 | consumed tokens: 1818230784 | elapsed time per iteration (s): 5.40 | learning rate: 6.916E-04 | global batch size: 512 | lm loss: 3.400775E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.753 | TFLOPs: 71.57 | +[default7]: iteration 1735/ 3814 | consumed samples: 888320 | consumed tokens: 1819279360 | elapsed time per iteration (s): 5.43 | learning rate: 6.912E-04 | global batch size: 512 | lm loss: 3.401177E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.255 | TFLOPs: 71.19 | +[default7]: iteration 1736/ 3814 | consumed samples: 888832 | consumed tokens: 1820327936 | elapsed time per iteration (s): 5.41 | learning rate: 6.908E-04 | global batch size: 512 | lm loss: 3.396019E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.677 | TFLOPs: 71.51 | +[default7]: iteration 1737/ 3814 | consumed samples: 889344 | consumed tokens: 1821376512 | elapsed time per iteration (s): 5.42 | learning rate: 6.904E-04 | global batch size: 512 | lm loss: 3.409710E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.546 | TFLOPs: 71.41 | +[default7]: iteration 1738/ 3814 | consumed samples: 889856 | consumed tokens: 1822425088 | elapsed time per iteration (s): 5.42 | learning rate: 6.900E-04 | global batch size: 512 | lm loss: 3.391613E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.495 | TFLOPs: 71.37 | +[default7]: iteration 1739/ 3814 | consumed samples: 890368 | consumed tokens: 1823473664 | elapsed time per iteration (s): 5.42 | learning rate: 6.896E-04 | global batch size: 512 | lm loss: 3.378768E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 1740/ 3814 | consumed samples: 890880 | consumed tokens: 1824522240 | elapsed time per iteration (s): 5.41 | learning rate: 6.892E-04 | global batch size: 512 | lm loss: 3.389790E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.602 | TFLOPs: 71.45 | +[default7]: iteration 1741/ 3814 | consumed samples: 891392 | consumed tokens: 1825570816 | elapsed time per iteration (s): 5.43 | learning rate: 6.888E-04 | global batch size: 512 | lm loss: 3.396095E+00 | loss scale: 32768.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 1742/ 3814 | consumed samples: 891904 | consumed tokens: 1826619392 | elapsed time per iteration (s): 5.41 | learning rate: 6.885E-04 | global batch size: 512 | lm loss: 3.414612E+00 | loss scale: 32768.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.612 | TFLOPs: 71.46 | +[default7]: iteration 1743/ 3814 | consumed samples: 892416 | consumed tokens: 1827667968 | elapsed time per iteration (s): 5.41 | learning rate: 6.881E-04 | global batch size: 512 | lm loss: 3.391124E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.715 | TFLOPs: 71.54 | +[default7]: iteration 1744/ 3814 | consumed samples: 892928 | consumed tokens: 1828716544 | elapsed time per iteration (s): 5.41 | learning rate: 6.877E-04 | global batch size: 512 | lm loss: 3.396655E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.586 | TFLOPs: 71.44 | +[default7]: iteration 1745/ 3814 | consumed samples: 893440 | consumed tokens: 1829765120 | elapsed time per iteration (s): 5.41 | learning rate: 6.873E-04 | global batch size: 512 | lm loss: 3.387468E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1746/ 3814 | consumed samples: 893952 | consumed tokens: 1830813696 | elapsed time per iteration (s): 5.40 | learning rate: 6.869E-04 | global batch size: 512 | lm loss: 3.389954E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 1747/ 3814 | consumed samples: 894464 | consumed tokens: 1831862272 | elapsed time per iteration (s): 5.40 | learning rate: 6.865E-04 | global batch size: 512 | lm loss: 3.393652E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.798 | TFLOPs: 71.60 | +[default7]: iteration 1748/ 3814 | consumed samples: 894976 | consumed tokens: 1832910848 | elapsed time per iteration (s): 5.41 | learning rate: 6.861E-04 | global batch size: 512 | lm loss: 3.419400E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1749/ 3814 | consumed samples: 895488 | consumed tokens: 1833959424 | elapsed time per iteration (s): 5.41 | learning rate: 6.857E-04 | global batch size: 512 | lm loss: 3.369953E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default0]:saving checkpoint at iteration 1750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 08:57:52,191] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1750 is begin to save! +[default0]:[2023-02-16 08:57:52,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_01-model_01-model_states.pt... +[default7]: iteration 1750/ 3814 | consumed samples: 896000 | consumed tokens: 1835008000 | elapsed time per iteration (s): 5.42 | learning rate: 6.853E-04 | global batch size: 512 | lm loss: 3.366114E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.527 | TFLOPs: 71.40 | +[default1]:[2023-02-16 08:57:52,347] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,351] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,351] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_04-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:52,410] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,410] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,405] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,406] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_05-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:52,460] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,460] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_06-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,468] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:52,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,524] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,515] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,515] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_07-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:52,571] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,572] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_08-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,579] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,580] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,631] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,632] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,637] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,637] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_09-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:52,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,693] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,687] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,687] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:52,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,742] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,749] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,749] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_11-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:52,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,805] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,798] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,798] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:52,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,854] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:52,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,919] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:52,911] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,912] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_14-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:52,965] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:52,966] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:52,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:52,973] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,021] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_16-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,076] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,076] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,029] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,029] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_16-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,131] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,139] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,139] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,194] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,195] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,187] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,242] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,251] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_20-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,306] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,307] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,298] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,298] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,353] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,354] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,361] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,362] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,420] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,421] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,411] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,411] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_23-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,468] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,469] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,479] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,480] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,528] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,538] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,596] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,586] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_26-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,652] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,653] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,707] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,708] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,696] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,696] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_28-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,751] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,769] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_29-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,826] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_29-model_00-model_states.pt. +[default1]:[2023-02-16 08:57:53,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,805] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_30-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,860] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,860] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,826] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,883] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,884] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:53,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,918] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:53,972] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:53,973] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:53,940] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,940] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:53,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:53,997] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:54,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:54,041] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 08:57:54,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:54,065] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:54,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:54,106] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:54,167] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:54,167] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 08:57:54,168] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 08:57:54,169] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_01_model_states.pt +[default1]:[2023-02-16 08:57:54,169] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 08:57:54,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_01_model_states.pt. +[default0]:[2023-02-16 08:57:54,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:54,132] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 08:57:54,198] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:54,198] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_37-model_00-model_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:57:54,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 08:57:54,200] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_00_model_states.pt +[default0]:[2023-02-16 08:57:54,200] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 08:57:54,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/mp_rank_00_model_states.pt. +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 08:57:54,698] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,698] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,698] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,768] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,768] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,739] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,768] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,768] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,768] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,766] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,767] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,725] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,726] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,726] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,776] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,776] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,720] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,720] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,739] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,736] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,737] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,737] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,741] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,741] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,780] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,780] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,763] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,763] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,808] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,808] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,825] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,825] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,825] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,825] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,804] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,804] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,804] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,847] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,847] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,798] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,798] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,856] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,856] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,854] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,854] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,886] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,887] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,887] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,859] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,859] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,859] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,810] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,810] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,815] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,815] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,815] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,838] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,838] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,893] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,893] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,854] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,854] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,854] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,862] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,863] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,808] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,808] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,808] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,805] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,805] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,805] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,880] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,880] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,880] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,885] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,885] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,853] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,853] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,864] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,864] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,889] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,889] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,900] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,900] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,910] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,910] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,885] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,885] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,877] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,877] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,877] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,835] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,835] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,835] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,859] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,859] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,913] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,914] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,850] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,850] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,913] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,913] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,863] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,863] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,876] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,876] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,876] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,858] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,858] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,918] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,918] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,905] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,887] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,887] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:54,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:54,921] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:54,921] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,910] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,910] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,934] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,934] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default3]:[2023-02-16 08:57:54,892] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 08:57:54,892] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 08:57:54,892] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default5]:[2023-02-16 08:57:54,915] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:54,915] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:54,916] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,925] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,925] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,925] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default1]:[2023-02-16 08:57:54,919] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 08:57:54,919] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 08:57:54,919] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default4]:[2023-02-16 08:57:54,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 08:57:54,921] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 08:57:54,922] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]:[2023-02-16 08:57:54,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 08:57:54,947] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 08:57:54,947] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default2]:[2023-02-16 08:57:54,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 08:57:54,927] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 08:57:54,927] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default6]:[2023-02-16 08:57:54,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 08:57:54,968] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 08:57:54,968] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:[2023-02-16 08:57:55,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 08:57:55,001] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 08:57:55,001] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default7]:time (ms) | save-checkpoint: 2895.93 +[default5]:[2023-02-16 08:57:55,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 08:57:55,086] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step1750/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 08:57:55,086] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1750 is ready now! +[default0]: successfully saved checkpoint at iteration 1750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]: iteration 1751/ 3814 | consumed samples: 896512 | consumed tokens: 1836056576 | elapsed time per iteration (s): 8.33 | learning rate: 6.850E-04 | global batch size: 512 | lm loss: 3.377398E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.472 | TFLOPs: 46.43 | +[default7]: iteration 1752/ 3814 | consumed samples: 897024 | consumed tokens: 1837105152 | elapsed time per iteration (s): 5.42 | learning rate: 6.846E-04 | global batch size: 512 | lm loss: 3.363276E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.402 | TFLOPs: 71.30 | +[default7]: iteration 1753/ 3814 | consumed samples: 897536 | consumed tokens: 1838153728 | elapsed time per iteration (s): 5.41 | learning rate: 6.842E-04 | global batch size: 512 | lm loss: 3.369870E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 1754/ 3814 | consumed samples: 898048 | consumed tokens: 1839202304 | elapsed time per iteration (s): 5.41 | learning rate: 6.838E-04 | global batch size: 512 | lm loss: 3.397849E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.689 | TFLOPs: 71.52 | +[default7]: iteration 1755/ 3814 | consumed samples: 898560 | consumed tokens: 1840250880 | elapsed time per iteration (s): 5.42 | learning rate: 6.834E-04 | global batch size: 512 | lm loss: 3.393461E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.460 | TFLOPs: 71.35 | +[default7]: iteration 1756/ 3814 | consumed samples: 899072 | consumed tokens: 1841299456 | elapsed time per iteration (s): 5.43 | learning rate: 6.830E-04 | global batch size: 512 | lm loss: 3.384647E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.240 | TFLOPs: 71.18 | +[default7]: iteration 1757/ 3814 | consumed samples: 899584 | consumed tokens: 1842348032 | elapsed time per iteration (s): 5.45 | learning rate: 6.826E-04 | global batch size: 512 | lm loss: 3.401488E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.942 | TFLOPs: 70.96 | +[default7]: iteration 1758/ 3814 | consumed samples: 900096 | consumed tokens: 1843396608 | elapsed time per iteration (s): 5.43 | learning rate: 6.822E-04 | global batch size: 512 | lm loss: 3.355487E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 1759/ 3814 | consumed samples: 900608 | consumed tokens: 1844445184 | elapsed time per iteration (s): 5.43 | learning rate: 6.818E-04 | global batch size: 512 | lm loss: 3.388938E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.260 | TFLOPs: 71.20 | +[default7]: iteration 1760/ 3814 | consumed samples: 901120 | consumed tokens: 1845493760 | elapsed time per iteration (s): 5.43 | learning rate: 6.814E-04 | global batch size: 512 | lm loss: 3.378802E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.292 | TFLOPs: 71.22 | +[default7]: iteration 1761/ 3814 | consumed samples: 901632 | consumed tokens: 1846542336 | elapsed time per iteration (s): 5.45 | learning rate: 6.810E-04 | global batch size: 512 | lm loss: 3.361322E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.897 | TFLOPs: 70.92 | +[default7]: iteration 1762/ 3814 | consumed samples: 902144 | consumed tokens: 1847590912 | elapsed time per iteration (s): 5.44 | learning rate: 6.807E-04 | global batch size: 512 | lm loss: 3.383399E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.200 | TFLOPs: 71.15 | +[default7]: iteration 1763/ 3814 | consumed samples: 902656 | consumed tokens: 1848639488 | elapsed time per iteration (s): 5.44 | learning rate: 6.803E-04 | global batch size: 512 | lm loss: 3.428860E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.051 | TFLOPs: 71.04 | +[default7]: iteration 1764/ 3814 | consumed samples: 903168 | consumed tokens: 1849688064 | elapsed time per iteration (s): 5.44 | learning rate: 6.799E-04 | global batch size: 512 | lm loss: 3.393203E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.160 | TFLOPs: 71.12 | +[default7]: iteration 1765/ 3814 | consumed samples: 903680 | consumed tokens: 1850736640 | elapsed time per iteration (s): 5.43 | learning rate: 6.795E-04 | global batch size: 512 | lm loss: 3.373347E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.278 | TFLOPs: 71.21 | +[default7]: iteration 1766/ 3814 | consumed samples: 904192 | consumed tokens: 1851785216 | elapsed time per iteration (s): 5.43 | learning rate: 6.791E-04 | global batch size: 512 | lm loss: 3.353467E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.287 | TFLOPs: 71.22 | +[default7]: iteration 1767/ 3814 | consumed samples: 904704 | consumed tokens: 1852833792 | elapsed time per iteration (s): 5.43 | learning rate: 6.787E-04 | global batch size: 512 | lm loss: 3.385717E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.350 | TFLOPs: 71.26 | +[default7]: iteration 1768/ 3814 | consumed samples: 905216 | consumed tokens: 1853882368 | elapsed time per iteration (s): 5.44 | learning rate: 6.783E-04 | global batch size: 512 | lm loss: 3.413446E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.055 | TFLOPs: 71.04 | +[default7]: iteration 1769/ 3814 | consumed samples: 905728 | consumed tokens: 1854930944 | elapsed time per iteration (s): 5.43 | learning rate: 6.779E-04 | global batch size: 512 | lm loss: 3.405269E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.210 | TFLOPs: 71.16 | +[default7]: iteration 1770/ 3814 | consumed samples: 906240 | consumed tokens: 1855979520 | elapsed time per iteration (s): 5.43 | learning rate: 6.775E-04 | global batch size: 512 | lm loss: 3.359902E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.314 | TFLOPs: 71.24 | +[default7]: iteration 1771/ 3814 | consumed samples: 906752 | consumed tokens: 1857028096 | elapsed time per iteration (s): 5.43 | learning rate: 6.771E-04 | global batch size: 512 | lm loss: 3.372999E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.231 | TFLOPs: 71.17 | +[default7]: iteration 1772/ 3814 | consumed samples: 907264 | consumed tokens: 1858076672 | elapsed time per iteration (s): 5.43 | learning rate: 6.767E-04 | global batch size: 512 | lm loss: 3.389331E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.236 | TFLOPs: 71.18 | +[default7]: iteration 1773/ 3814 | consumed samples: 907776 | consumed tokens: 1859125248 | elapsed time per iteration (s): 5.47 | learning rate: 6.763E-04 | global batch size: 512 | lm loss: 3.381717E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.595 | TFLOPs: 70.69 | +[default7]: iteration 1774/ 3814 | consumed samples: 908288 | consumed tokens: 1860173824 | elapsed time per iteration (s): 5.43 | learning rate: 6.759E-04 | global batch size: 512 | lm loss: 3.348802E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.354 | TFLOPs: 71.27 | +[default7]: iteration 1775/ 3814 | consumed samples: 908800 | consumed tokens: 1861222400 | elapsed time per iteration (s): 5.42 | learning rate: 6.756E-04 | global batch size: 512 | lm loss: 3.367898E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.498 | TFLOPs: 71.38 | +[default7]: iteration 1776/ 3814 | consumed samples: 909312 | consumed tokens: 1862270976 | elapsed time per iteration (s): 5.43 | learning rate: 6.752E-04 | global batch size: 512 | lm loss: 3.357261E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.324 | TFLOPs: 71.24 | +[default7]: iteration 1777/ 3814 | consumed samples: 909824 | consumed tokens: 1863319552 | elapsed time per iteration (s): 5.43 | learning rate: 6.748E-04 | global batch size: 512 | lm loss: 3.378910E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.317 | TFLOPs: 71.24 | +[default7]: iteration 1778/ 3814 | consumed samples: 910336 | consumed tokens: 1864368128 | elapsed time per iteration (s): 5.41 | learning rate: 6.744E-04 | global batch size: 512 | lm loss: 3.386364E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.560 | TFLOPs: 71.42 | +[default7]: iteration 1779/ 3814 | consumed samples: 910848 | consumed tokens: 1865416704 | elapsed time per iteration (s): 5.43 | learning rate: 6.740E-04 | global batch size: 512 | lm loss: 3.352241E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.330 | TFLOPs: 71.25 | +[default7]: iteration 1780/ 3814 | consumed samples: 911360 | consumed tokens: 1866465280 | elapsed time per iteration (s): 5.44 | learning rate: 6.736E-04 | global batch size: 512 | lm loss: 3.352272E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.155 | TFLOPs: 71.12 | +[default7]: iteration 1781/ 3814 | consumed samples: 911872 | consumed tokens: 1867513856 | elapsed time per iteration (s): 5.42 | learning rate: 6.732E-04 | global batch size: 512 | lm loss: 3.383104E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.405 | TFLOPs: 71.31 | +[default7]: iteration 1782/ 3814 | consumed samples: 912384 | consumed tokens: 1868562432 | elapsed time per iteration (s): 5.43 | learning rate: 6.728E-04 | global batch size: 512 | lm loss: 3.365280E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1783/ 3814 | consumed samples: 912896 | consumed tokens: 1869611008 | elapsed time per iteration (s): 5.42 | learning rate: 6.724E-04 | global batch size: 512 | lm loss: 3.369872E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.499 | TFLOPs: 71.38 | +[default7]: iteration 1784/ 3814 | consumed samples: 913408 | consumed tokens: 1870659584 | elapsed time per iteration (s): 5.42 | learning rate: 6.720E-04 | global batch size: 512 | lm loss: 3.363634E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.405 | TFLOPs: 71.30 | +[default7]: iteration 1785/ 3814 | consumed samples: 913920 | consumed tokens: 1871708160 | elapsed time per iteration (s): 5.43 | learning rate: 6.716E-04 | global batch size: 512 | lm loss: 3.382450E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.215 | TFLOPs: 71.16 | +[default7]: iteration 1786/ 3814 | consumed samples: 914432 | consumed tokens: 1872756736 | elapsed time per iteration (s): 5.43 | learning rate: 6.712E-04 | global batch size: 512 | lm loss: 3.383596E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.221 | TFLOPs: 71.17 | +[default7]: iteration 1787/ 3814 | consumed samples: 914944 | consumed tokens: 1873805312 | elapsed time per iteration (s): 5.42 | learning rate: 6.708E-04 | global batch size: 512 | lm loss: 3.362179E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.505 | TFLOPs: 71.38 | +[default7]: iteration 1788/ 3814 | consumed samples: 915456 | consumed tokens: 1874853888 | elapsed time per iteration (s): 5.42 | learning rate: 6.704E-04 | global batch size: 512 | lm loss: 3.362018E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.442 | TFLOPs: 71.33 | +[default7]: iteration 1789/ 3814 | consumed samples: 915968 | consumed tokens: 1875902464 | elapsed time per iteration (s): 5.44 | learning rate: 6.700E-04 | global batch size: 512 | lm loss: 3.367633E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.079 | TFLOPs: 71.06 | +[default7]: iteration 1790/ 3814 | consumed samples: 916480 | consumed tokens: 1876951040 | elapsed time per iteration (s): 5.43 | learning rate: 6.697E-04 | global batch size: 512 | lm loss: 3.400667E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.211 | TFLOPs: 71.16 | +[default7]: iteration 1791/ 3814 | consumed samples: 916992 | consumed tokens: 1877999616 | elapsed time per iteration (s): 5.44 | learning rate: 6.693E-04 | global batch size: 512 | lm loss: 3.370756E+00 | loss scale: 32768.0 | grad norm: 0.131 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.146 | TFLOPs: 71.11 | +[default7]: iteration 1792/ 3814 | consumed samples: 917504 | consumed tokens: 1879048192 | elapsed time per iteration (s): 5.46 | learning rate: 6.689E-04 | global batch size: 512 | lm loss: 3.405200E+00 | loss scale: 32768.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.780 | TFLOPs: 70.83 | +[default7]: iteration 1793/ 3814 | consumed samples: 918016 | consumed tokens: 1880096768 | elapsed time per iteration (s): 5.45 | learning rate: 6.685E-04 | global batch size: 512 | lm loss: 3.365144E+00 | loss scale: 32768.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.889 | TFLOPs: 70.92 | +[default7]: iteration 1794/ 3814 | consumed samples: 918528 | consumed tokens: 1881145344 | elapsed time per iteration (s): 5.44 | learning rate: 6.681E-04 | global batch size: 512 | lm loss: 3.380971E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.200 | TFLOPs: 71.15 | +[default7]: iteration 1795/ 3814 | consumed samples: 919040 | consumed tokens: 1882193920 | elapsed time per iteration (s): 5.43 | learning rate: 6.677E-04 | global batch size: 512 | lm loss: 3.347746E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.267 | TFLOPs: 71.20 | +[default7]: iteration 1796/ 3814 | consumed samples: 919552 | consumed tokens: 1883242496 | elapsed time per iteration (s): 5.42 | learning rate: 6.673E-04 | global batch size: 512 | lm loss: 3.357394E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 1797/ 3814 | consumed samples: 920064 | consumed tokens: 1884291072 | elapsed time per iteration (s): 5.41 | learning rate: 6.669E-04 | global batch size: 512 | lm loss: 3.369630E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 1798/ 3814 | consumed samples: 920576 | consumed tokens: 1885339648 | elapsed time per iteration (s): 5.42 | learning rate: 6.665E-04 | global batch size: 512 | lm loss: 3.354225E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.393 | TFLOPs: 71.30 | +[default7]: iteration 1799/ 3814 | consumed samples: 921088 | consumed tokens: 1886388224 | elapsed time per iteration (s): 5.42 | learning rate: 6.661E-04 | global batch size: 512 | lm loss: 3.397034E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.436 | TFLOPs: 71.33 | +[default7]: iteration 1800/ 3814 | consumed samples: 921600 | consumed tokens: 1887436800 | elapsed time per iteration (s): 5.43 | learning rate: 6.657E-04 | global batch size: 512 | lm loss: 3.380838E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 1801/ 3814 | consumed samples: 922112 | consumed tokens: 1888485376 | elapsed time per iteration (s): 5.42 | learning rate: 6.653E-04 | global batch size: 512 | lm loss: 3.377165E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.401 | TFLOPs: 71.30 | +[default7]: iteration 1802/ 3814 | consumed samples: 922624 | consumed tokens: 1889533952 | elapsed time per iteration (s): 5.43 | learning rate: 6.649E-04 | global batch size: 512 | lm loss: 3.379674E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.219 | TFLOPs: 71.16 | +[default7]: iteration 1803/ 3814 | consumed samples: 923136 | consumed tokens: 1890582528 | elapsed time per iteration (s): 5.42 | learning rate: 6.645E-04 | global batch size: 512 | lm loss: 3.374928E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.483 | TFLOPs: 71.36 | +[default7]: iteration 1804/ 3814 | consumed samples: 923648 | consumed tokens: 1891631104 | elapsed time per iteration (s): 5.41 | learning rate: 6.641E-04 | global batch size: 512 | lm loss: 3.386625E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.633 | TFLOPs: 71.48 | +[default7]: iteration 1805/ 3814 | consumed samples: 924160 | consumed tokens: 1892679680 | elapsed time per iteration (s): 5.40 | learning rate: 6.637E-04 | global batch size: 512 | lm loss: 3.387190E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.786 | TFLOPs: 71.59 | +[default7]: iteration 1806/ 3814 | consumed samples: 924672 | consumed tokens: 1893728256 | elapsed time per iteration (s): 5.40 | learning rate: 6.633E-04 | global batch size: 512 | lm loss: 3.371259E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.800 | TFLOPs: 71.60 | +[default7]: iteration 1807/ 3814 | consumed samples: 925184 | consumed tokens: 1894776832 | elapsed time per iteration (s): 5.41 | learning rate: 6.629E-04 | global batch size: 512 | lm loss: 3.355441E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 1808/ 3814 | consumed samples: 925696 | consumed tokens: 1895825408 | elapsed time per iteration (s): 5.39 | learning rate: 6.625E-04 | global batch size: 512 | lm loss: 3.356882E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.942 | TFLOPs: 71.71 | +[default7]: iteration 1809/ 3814 | consumed samples: 926208 | consumed tokens: 1896873984 | elapsed time per iteration (s): 5.41 | learning rate: 6.621E-04 | global batch size: 512 | lm loss: 3.382745E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.635 | TFLOPs: 71.48 | +[default7]: iteration 1810/ 3814 | consumed samples: 926720 | consumed tokens: 1897922560 | elapsed time per iteration (s): 5.42 | learning rate: 6.618E-04 | global batch size: 512 | lm loss: 3.313962E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.459 | TFLOPs: 71.35 | +[default7]: iteration 1811/ 3814 | consumed samples: 927232 | consumed tokens: 1898971136 | elapsed time per iteration (s): 5.41 | learning rate: 6.614E-04 | global batch size: 512 | lm loss: 3.376925E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.681 | TFLOPs: 71.51 | +[default7]: iteration 1812/ 3814 | consumed samples: 927744 | consumed tokens: 1900019712 | elapsed time per iteration (s): 5.41 | learning rate: 6.610E-04 | global batch size: 512 | lm loss: 3.379969E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 1813/ 3814 | consumed samples: 928256 | consumed tokens: 1901068288 | elapsed time per iteration (s): 5.41 | learning rate: 6.606E-04 | global batch size: 512 | lm loss: 3.352802E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.581 | TFLOPs: 71.44 | +[default7]: iteration 1814/ 3814 | consumed samples: 928768 | consumed tokens: 1902116864 | elapsed time per iteration (s): 5.41 | learning rate: 6.602E-04 | global batch size: 512 | lm loss: 3.343930E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 1815/ 3814 | consumed samples: 929280 | consumed tokens: 1903165440 | elapsed time per iteration (s): 5.42 | learning rate: 6.598E-04 | global batch size: 512 | lm loss: 3.346148E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.509 | TFLOPs: 71.38 | +[default7]: iteration 1816/ 3814 | consumed samples: 929792 | consumed tokens: 1904214016 | elapsed time per iteration (s): 5.40 | learning rate: 6.594E-04 | global batch size: 512 | lm loss: 3.363312E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.834 | TFLOPs: 71.63 | +[default7]: iteration 1817/ 3814 | consumed samples: 930304 | consumed tokens: 1905262592 | elapsed time per iteration (s): 5.39 | learning rate: 6.590E-04 | global batch size: 512 | lm loss: 3.378754E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.928 | TFLOPs: 71.70 | +[default7]: iteration 1818/ 3814 | consumed samples: 930816 | consumed tokens: 1906311168 | elapsed time per iteration (s): 5.40 | learning rate: 6.586E-04 | global batch size: 512 | lm loss: 3.367954E+00 | loss scale: 32768.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.746 | TFLOPs: 71.56 | +[default7]: iteration 1819/ 3814 | consumed samples: 931328 | consumed tokens: 1907359744 | elapsed time per iteration (s): 5.40 | learning rate: 6.582E-04 | global batch size: 512 | lm loss: 3.385093E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.747 | TFLOPs: 71.56 | +[default7]: iteration 1820/ 3814 | consumed samples: 931840 | consumed tokens: 1908408320 | elapsed time per iteration (s): 5.40 | learning rate: 6.578E-04 | global batch size: 512 | lm loss: 3.347778E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.899 | TFLOPs: 71.68 | +[default7]: iteration 1821/ 3814 | consumed samples: 932352 | consumed tokens: 1909456896 | elapsed time per iteration (s): 5.40 | learning rate: 6.574E-04 | global batch size: 512 | lm loss: 3.360349E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.756 | TFLOPs: 71.57 | +[default7]: iteration 1822/ 3814 | consumed samples: 932864 | consumed tokens: 1910505472 | elapsed time per iteration (s): 5.40 | learning rate: 6.570E-04 | global batch size: 512 | lm loss: 3.360657E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.64 | +[default7]: iteration 1823/ 3814 | consumed samples: 933376 | consumed tokens: 1911554048 | elapsed time per iteration (s): 5.40 | learning rate: 6.566E-04 | global batch size: 512 | lm loss: 3.366920E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.758 | TFLOPs: 71.57 | +[default7]: iteration 1824/ 3814 | consumed samples: 933888 | consumed tokens: 1912602624 | elapsed time per iteration (s): 5.40 | learning rate: 6.562E-04 | global batch size: 512 | lm loss: 3.345996E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.756 | TFLOPs: 71.57 | +[default7]: iteration 1825/ 3814 | consumed samples: 934400 | consumed tokens: 1913651200 | elapsed time per iteration (s): 5.42 | learning rate: 6.558E-04 | global batch size: 512 | lm loss: 3.350820E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.400 | TFLOPs: 71.30 | +[default7]: iteration 1826/ 3814 | consumed samples: 934912 | consumed tokens: 1914699776 | elapsed time per iteration (s): 5.40 | learning rate: 6.554E-04 | global batch size: 512 | lm loss: 3.362351E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.65 | +[default7]: iteration 1827/ 3814 | consumed samples: 935424 | consumed tokens: 1915748352 | elapsed time per iteration (s): 5.42 | learning rate: 6.550E-04 | global batch size: 512 | lm loss: 3.371299E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.539 | TFLOPs: 71.41 | +[default7]: iteration 1828/ 3814 | consumed samples: 935936 | consumed tokens: 1916796928 | elapsed time per iteration (s): 5.43 | learning rate: 6.546E-04 | global batch size: 512 | lm loss: 3.365680E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.356 | TFLOPs: 71.27 | +[default7]: iteration 1829/ 3814 | consumed samples: 936448 | consumed tokens: 1917845504 | elapsed time per iteration (s): 5.41 | learning rate: 6.542E-04 | global batch size: 512 | lm loss: 3.355664E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.675 | TFLOPs: 71.51 | +[default7]: iteration 1830/ 3814 | consumed samples: 936960 | consumed tokens: 1918894080 | elapsed time per iteration (s): 5.42 | learning rate: 6.538E-04 | global batch size: 512 | lm loss: 3.372277E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.413 | TFLOPs: 71.31 | +[default7]: iteration 1831/ 3814 | consumed samples: 937472 | consumed tokens: 1919942656 | elapsed time per iteration (s): 5.40 | learning rate: 6.534E-04 | global batch size: 512 | lm loss: 3.370849E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.771 | TFLOPs: 71.58 | +[default7]: iteration 1832/ 3814 | consumed samples: 937984 | consumed tokens: 1920991232 | elapsed time per iteration (s): 5.41 | learning rate: 6.530E-04 | global batch size: 512 | lm loss: 3.347429E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.597 | TFLOPs: 71.45 | +[default7]: iteration 1833/ 3814 | consumed samples: 938496 | consumed tokens: 1922039808 | elapsed time per iteration (s): 5.41 | learning rate: 6.526E-04 | global batch size: 512 | lm loss: 3.360515E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.664 | TFLOPs: 71.50 | +[default7]: iteration 1834/ 3814 | consumed samples: 939008 | consumed tokens: 1923088384 | elapsed time per iteration (s): 5.43 | learning rate: 6.522E-04 | global batch size: 512 | lm loss: 3.339421E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.275 | TFLOPs: 71.21 | +[default7]: iteration 1835/ 3814 | consumed samples: 939520 | consumed tokens: 1924136960 | elapsed time per iteration (s): 5.42 | learning rate: 6.518E-04 | global batch size: 512 | lm loss: 3.389043E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.438 | TFLOPs: 71.33 | +[default7]: iteration 1836/ 3814 | consumed samples: 940032 | consumed tokens: 1925185536 | elapsed time per iteration (s): 5.41 | learning rate: 6.514E-04 | global batch size: 512 | lm loss: 3.378322E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.563 | TFLOPs: 71.42 | +[default7]: iteration 1837/ 3814 | consumed samples: 940544 | consumed tokens: 1926234112 | elapsed time per iteration (s): 5.42 | learning rate: 6.510E-04 | global batch size: 512 | lm loss: 3.390729E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.495 | TFLOPs: 71.37 | +[default7]: iteration 1838/ 3814 | consumed samples: 941056 | consumed tokens: 1927282688 | elapsed time per iteration (s): 5.40 | learning rate: 6.506E-04 | global batch size: 512 | lm loss: 3.357590E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.740 | TFLOPs: 71.56 | +[default7]: iteration 1839/ 3814 | consumed samples: 941568 | consumed tokens: 1928331264 | elapsed time per iteration (s): 5.42 | learning rate: 6.502E-04 | global batch size: 512 | lm loss: 3.354673E+00 | loss scale: 32768.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.427 | TFLOPs: 71.32 | +[default7]: iteration 1840/ 3814 | consumed samples: 942080 | consumed tokens: 1929379840 | elapsed time per iteration (s): 5.42 | learning rate: 6.498E-04 | global batch size: 512 | lm loss: 3.341368E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.493 | TFLOPs: 71.37 | +[default7]: iteration 1841/ 3814 | consumed samples: 942592 | consumed tokens: 1930428416 | elapsed time per iteration (s): 5.42 | learning rate: 6.494E-04 | global batch size: 512 | lm loss: 3.348778E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.506 | TFLOPs: 71.38 | +[default7]: iteration 1842/ 3814 | consumed samples: 943104 | consumed tokens: 1931476992 | elapsed time per iteration (s): 5.43 | learning rate: 6.490E-04 | global batch size: 512 | lm loss: 3.373226E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.312 | TFLOPs: 71.23 | +[default7]: iteration 1843/ 3814 | consumed samples: 943616 | consumed tokens: 1932525568 | elapsed time per iteration (s): 5.43 | learning rate: 6.486E-04 | global batch size: 512 | lm loss: 3.373309E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.329 | TFLOPs: 71.25 | +[default7]: iteration 1844/ 3814 | consumed samples: 944128 | consumed tokens: 1933574144 | elapsed time per iteration (s): 5.43 | learning rate: 6.482E-04 | global batch size: 512 | lm loss: 3.370559E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.322 | TFLOPs: 71.24 | +[default7]: iteration 1845/ 3814 | consumed samples: 944640 | consumed tokens: 1934622720 | elapsed time per iteration (s): 5.44 | learning rate: 6.478E-04 | global batch size: 512 | lm loss: 3.339517E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.187 | TFLOPs: 71.14 | +[default7]: iteration 1846/ 3814 | consumed samples: 945152 | consumed tokens: 1935671296 | elapsed time per iteration (s): 5.41 | learning rate: 6.474E-04 | global batch size: 512 | lm loss: 3.333567E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 1847/ 3814 | consumed samples: 945664 | consumed tokens: 1936719872 | elapsed time per iteration (s): 5.42 | learning rate: 6.470E-04 | global batch size: 512 | lm loss: 3.383898E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.445 | TFLOPs: 71.34 | +[default7]: iteration 1848/ 3814 | consumed samples: 946176 | consumed tokens: 1937768448 | elapsed time per iteration (s): 5.41 | learning rate: 6.466E-04 | global batch size: 512 | lm loss: 3.360735E+00 | loss scale: 32768.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.638 | TFLOPs: 71.48 | +[default7]: iteration 1849/ 3814 | consumed samples: 946688 | consumed tokens: 1938817024 | elapsed time per iteration (s): 5.43 | learning rate: 6.462E-04 | global batch size: 512 | lm loss: 3.379392E+00 | loss scale: 32768.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.256 | TFLOPs: 71.19 | +[default7]: iteration 1850/ 3814 | consumed samples: 947200 | consumed tokens: 1939865600 | elapsed time per iteration (s): 5.43 | learning rate: 6.458E-04 | global batch size: 512 | lm loss: 3.331256E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.333 | TFLOPs: 71.25 | +[default7]: iteration 1851/ 3814 | consumed samples: 947712 | consumed tokens: 1940914176 | elapsed time per iteration (s): 5.44 | learning rate: 6.454E-04 | global batch size: 512 | lm loss: 3.329037E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.190 | TFLOPs: 71.14 | +[default7]: iteration 1852/ 3814 | consumed samples: 948224 | consumed tokens: 1941962752 | elapsed time per iteration (s): 5.43 | learning rate: 6.450E-04 | global batch size: 512 | lm loss: 3.350693E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.320 | TFLOPs: 71.24 | +[default7]: iteration 1853/ 3814 | consumed samples: 948736 | consumed tokens: 1943011328 | elapsed time per iteration (s): 5.41 | learning rate: 6.446E-04 | global batch size: 512 | lm loss: 3.359582E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1854/ 3814 | consumed samples: 949248 | consumed tokens: 1944059904 | elapsed time per iteration (s): 5.42 | learning rate: 6.442E-04 | global batch size: 512 | lm loss: 3.368794E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.421 | TFLOPs: 71.32 | +[default7]: iteration 1855/ 3814 | consumed samples: 949760 | consumed tokens: 1945108480 | elapsed time per iteration (s): 5.42 | learning rate: 6.438E-04 | global batch size: 512 | lm loss: 3.348520E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.522 | TFLOPs: 71.39 | +[default7]: iteration 1856/ 3814 | consumed samples: 950272 | consumed tokens: 1946157056 | elapsed time per iteration (s): 5.44 | learning rate: 6.434E-04 | global batch size: 512 | lm loss: 3.326818E+00 | loss scale: 32768.0 | grad norm: 0.085 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.117 | TFLOPs: 71.09 | +[default7]: iteration 1857/ 3814 | consumed samples: 950784 | consumed tokens: 1947205632 | elapsed time per iteration (s): 5.44 | learning rate: 6.430E-04 | global batch size: 512 | lm loss: 3.327952E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.070 | TFLOPs: 71.05 | +[default7]: iteration 1858/ 3814 | consumed samples: 951296 | consumed tokens: 1948254208 | elapsed time per iteration (s): 5.43 | learning rate: 6.426E-04 | global batch size: 512 | lm loss: 3.372826E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.253 | TFLOPs: 71.19 | +[default7]: iteration 1859/ 3814 | consumed samples: 951808 | consumed tokens: 1949302784 | elapsed time per iteration (s): 5.44 | learning rate: 6.422E-04 | global batch size: 512 | lm loss: 3.327029E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.131 | TFLOPs: 71.10 | +[default7]: iteration 1860/ 3814 | consumed samples: 952320 | consumed tokens: 1950351360 | elapsed time per iteration (s): 5.42 | learning rate: 6.418E-04 | global batch size: 512 | lm loss: 3.350946E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.430 | TFLOPs: 71.32 | +[default7]: iteration 1861/ 3814 | consumed samples: 952832 | consumed tokens: 1951399936 | elapsed time per iteration (s): 5.44 | learning rate: 6.414E-04 | global batch size: 512 | lm loss: 3.328892E+00 | loss scale: 32768.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.160 | TFLOPs: 71.12 | +[default7]: iteration 1862/ 3814 | consumed samples: 953344 | consumed tokens: 1952448512 | elapsed time per iteration (s): 5.42 | learning rate: 6.410E-04 | global batch size: 512 | lm loss: 3.348829E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 1863/ 3814 | consumed samples: 953856 | consumed tokens: 1953497088 | elapsed time per iteration (s): 5.43 | learning rate: 6.406E-04 | global batch size: 512 | lm loss: 3.358550E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.314 | TFLOPs: 71.24 | +[default7]: iteration 1864/ 3814 | consumed samples: 954368 | consumed tokens: 1954545664 | elapsed time per iteration (s): 5.44 | learning rate: 6.402E-04 | global batch size: 512 | lm loss: 3.338687E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.117 | TFLOPs: 71.09 | +[default7]: iteration 1865/ 3814 | consumed samples: 954880 | consumed tokens: 1955594240 | elapsed time per iteration (s): 5.42 | learning rate: 6.398E-04 | global batch size: 512 | lm loss: 3.350168E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.499 | TFLOPs: 71.38 | +[default7]: iteration 1866/ 3814 | consumed samples: 955392 | consumed tokens: 1956642816 | elapsed time per iteration (s): 5.42 | learning rate: 6.394E-04 | global batch size: 512 | lm loss: 3.340009E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.470 | TFLOPs: 71.35 | +[default7]: iteration 1867/ 3814 | consumed samples: 955904 | consumed tokens: 1957691392 | elapsed time per iteration (s): 5.42 | learning rate: 6.390E-04 | global batch size: 512 | lm loss: 3.345943E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.382 | TFLOPs: 71.29 | +[default7]: iteration 1868/ 3814 | consumed samples: 956416 | consumed tokens: 1958739968 | elapsed time per iteration (s): 5.44 | learning rate: 6.386E-04 | global batch size: 512 | lm loss: 3.355304E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.128 | TFLOPs: 71.10 | +[default7]: iteration 1869/ 3814 | consumed samples: 956928 | consumed tokens: 1959788544 | elapsed time per iteration (s): 5.43 | learning rate: 6.382E-04 | global batch size: 512 | lm loss: 3.365321E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.222 | TFLOPs: 71.17 | +[default7]: iteration 1870/ 3814 | consumed samples: 957440 | consumed tokens: 1960837120 | elapsed time per iteration (s): 5.42 | learning rate: 6.378E-04 | global batch size: 512 | lm loss: 3.310566E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 1871/ 3814 | consumed samples: 957952 | consumed tokens: 1961885696 | elapsed time per iteration (s): 5.42 | learning rate: 6.374E-04 | global batch size: 512 | lm loss: 3.339988E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.429 | TFLOPs: 71.32 | +[default7]: iteration 1872/ 3814 | consumed samples: 958464 | consumed tokens: 1962934272 | elapsed time per iteration (s): 5.43 | learning rate: 6.370E-04 | global batch size: 512 | lm loss: 3.327482E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.228 | TFLOPs: 71.17 | +[default7]: iteration 1873/ 3814 | consumed samples: 958976 | consumed tokens: 1963982848 | elapsed time per iteration (s): 5.44 | learning rate: 6.366E-04 | global batch size: 512 | lm loss: 3.374193E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.144 | TFLOPs: 71.11 | +[default7]: iteration 1874/ 3814 | consumed samples: 959488 | consumed tokens: 1965031424 | elapsed time per iteration (s): 5.41 | learning rate: 6.362E-04 | global batch size: 512 | lm loss: 3.337171E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.644 | TFLOPs: 71.49 | +[default7]: iteration 1875/ 3814 | consumed samples: 960000 | consumed tokens: 1966080000 | elapsed time per iteration (s): 5.41 | learning rate: 6.358E-04 | global batch size: 512 | lm loss: 3.360159E+00 | loss scale: 32768.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.702 | TFLOPs: 71.53 | +[default7]: iteration 1876/ 3814 | consumed samples: 960512 | consumed tokens: 1967128576 | elapsed time per iteration (s): 5.41 | learning rate: 6.354E-04 | global batch size: 512 | lm loss: 3.350733E+00 | loss scale: 32768.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.701 | TFLOPs: 71.53 | +[default7]: iteration 1877/ 3814 | consumed samples: 961024 | consumed tokens: 1968177152 | elapsed time per iteration (s): 5.43 | learning rate: 6.350E-04 | global batch size: 512 | lm loss: 3.333128E+00 | loss scale: 32768.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.345 | TFLOPs: 71.26 | +[default7]: iteration 1878/ 3814 | consumed samples: 961536 | consumed tokens: 1969225728 | elapsed time per iteration (s): 5.42 | learning rate: 6.346E-04 | global batch size: 512 | lm loss: 3.327889E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.501 | TFLOPs: 71.38 | +[default7]: iteration 1879/ 3814 | consumed samples: 962048 | consumed tokens: 1970274304 | elapsed time per iteration (s): 5.43 | learning rate: 6.342E-04 | global batch size: 512 | lm loss: 3.332670E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.359 | TFLOPs: 71.27 | +[default7]: iteration 1880/ 3814 | consumed samples: 962560 | consumed tokens: 1971322880 | elapsed time per iteration (s): 5.42 | learning rate: 6.338E-04 | global batch size: 512 | lm loss: 3.373405E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.391 | TFLOPs: 71.29 | +[default7]: iteration 1881/ 3814 | consumed samples: 963072 | consumed tokens: 1972371456 | elapsed time per iteration (s): 5.44 | learning rate: 6.334E-04 | global batch size: 512 | lm loss: 3.362164E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.145 | TFLOPs: 71.11 | +[default7]: iteration 1882/ 3814 | consumed samples: 963584 | consumed tokens: 1973420032 | elapsed time per iteration (s): 5.41 | learning rate: 6.330E-04 | global batch size: 512 | lm loss: 3.341975E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.636 | TFLOPs: 71.48 | +[default7]: iteration 1883/ 3814 | consumed samples: 964096 | consumed tokens: 1974468608 | elapsed time per iteration (s): 5.42 | learning rate: 6.326E-04 | global batch size: 512 | lm loss: 3.331115E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.468 | TFLOPs: 71.35 | +[default7]: iteration 1884/ 3814 | consumed samples: 964608 | consumed tokens: 1975517184 | elapsed time per iteration (s): 5.41 | learning rate: 6.322E-04 | global batch size: 512 | lm loss: 3.351429E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default7]: iteration 1885/ 3814 | consumed samples: 965120 | consumed tokens: 1976565760 | elapsed time per iteration (s): 5.42 | learning rate: 6.318E-04 | global batch size: 512 | lm loss: 3.363095E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.382 | TFLOPs: 71.29 | +[default7]: iteration 1886/ 3814 | consumed samples: 965632 | consumed tokens: 1977614336 | elapsed time per iteration (s): 5.43 | learning rate: 6.314E-04 | global batch size: 512 | lm loss: 3.337886E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.215 | TFLOPs: 71.16 | +[default7]: iteration 1887/ 3814 | consumed samples: 966144 | consumed tokens: 1978662912 | elapsed time per iteration (s): 5.42 | learning rate: 6.310E-04 | global batch size: 512 | lm loss: 3.356778E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.479 | TFLOPs: 71.36 | +[default7]: iteration 1888/ 3814 | consumed samples: 966656 | consumed tokens: 1979711488 | elapsed time per iteration (s): 5.42 | learning rate: 6.306E-04 | global batch size: 512 | lm loss: 3.356489E+00 | loss scale: 32768.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.533 | TFLOPs: 71.40 | +[default7]: iteration 1889/ 3814 | consumed samples: 967168 | consumed tokens: 1980760064 | elapsed time per iteration (s): 5.43 | learning rate: 6.302E-04 | global batch size: 512 | lm loss: 3.332723E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.244 | TFLOPs: 71.18 | +[default7]: iteration 1890/ 3814 | consumed samples: 967680 | consumed tokens: 1981808640 | elapsed time per iteration (s): 5.44 | learning rate: 6.298E-04 | global batch size: 512 | lm loss: 3.351868E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.107 | TFLOPs: 71.08 | +[default7]: iteration 1891/ 3814 | consumed samples: 968192 | consumed tokens: 1982857216 | elapsed time per iteration (s): 5.42 | learning rate: 6.294E-04 | global batch size: 512 | lm loss: 3.335995E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 1892/ 3814 | consumed samples: 968704 | consumed tokens: 1983905792 | elapsed time per iteration (s): 5.43 | learning rate: 6.290E-04 | global batch size: 512 | lm loss: 3.371826E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.243 | TFLOPs: 71.18 | +[default7]: iteration 1893/ 3814 | consumed samples: 969216 | consumed tokens: 1984954368 | elapsed time per iteration (s): 5.43 | learning rate: 6.286E-04 | global batch size: 512 | lm loss: 3.323187E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.371 | TFLOPs: 71.28 | +[default7]: iteration 1894/ 3814 | consumed samples: 969728 | consumed tokens: 1986002944 | elapsed time per iteration (s): 5.42 | learning rate: 6.282E-04 | global batch size: 512 | lm loss: 3.339587E+00 | loss scale: 32768.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.448 | TFLOPs: 71.34 | +[default7]: iteration 1895/ 3814 | consumed samples: 970240 | consumed tokens: 1987051520 | elapsed time per iteration (s): 5.43 | learning rate: 6.278E-04 | global batch size: 512 | lm loss: 3.338011E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.355 | TFLOPs: 71.27 | +[default7]: iteration 1896/ 3814 | consumed samples: 970752 | consumed tokens: 1988100096 | elapsed time per iteration (s): 5.43 | learning rate: 6.274E-04 | global batch size: 512 | lm loss: 3.364685E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.242 | TFLOPs: 71.18 | +[default7]: iteration 1897/ 3814 | consumed samples: 971264 | consumed tokens: 1989148672 | elapsed time per iteration (s): 5.42 | learning rate: 6.270E-04 | global batch size: 512 | lm loss: 3.342361E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 1898/ 3814 | consumed samples: 971776 | consumed tokens: 1990197248 | elapsed time per iteration (s): 5.44 | learning rate: 6.266E-04 | global batch size: 512 | lm loss: 3.355081E+00 | loss scale: 32768.0 | grad norm: 0.082 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.177 | TFLOPs: 71.13 | +[default7]: iteration 1899/ 3814 | consumed samples: 972288 | consumed tokens: 1991245824 | elapsed time per iteration (s): 5.44 | learning rate: 6.262E-04 | global batch size: 512 | lm loss: 3.329142E+00 | loss scale: 32768.0 | grad norm: 0.084 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.173 | TFLOPs: 71.13 | +[default7]: iteration 1900/ 3814 | consumed samples: 972800 | consumed tokens: 1992294400 | elapsed time per iteration (s): 5.43 | learning rate: 6.258E-04 | global batch size: 512 | lm loss: 3.317909E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.368 | TFLOPs: 71.28 | +[default7]: iteration 1901/ 3814 | consumed samples: 973312 | consumed tokens: 1993342976 | elapsed time per iteration (s): 5.41 | learning rate: 6.254E-04 | global batch size: 512 | lm loss: 3.317059E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.628 | TFLOPs: 71.47 | +[default7]: iteration 1902/ 3814 | consumed samples: 973824 | consumed tokens: 1994391552 | elapsed time per iteration (s): 5.45 | learning rate: 6.250E-04 | global batch size: 512 | lm loss: 3.342791E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.887 | TFLOPs: 70.91 | +[default7]: iteration 1903/ 3814 | consumed samples: 974336 | consumed tokens: 1995440128 | elapsed time per iteration (s): 5.43 | learning rate: 6.246E-04 | global batch size: 512 | lm loss: 3.336322E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.346 | TFLOPs: 71.26 | +[default7]: iteration 1904/ 3814 | consumed samples: 974848 | consumed tokens: 1996488704 | elapsed time per iteration (s): 5.43 | learning rate: 6.242E-04 | global batch size: 512 | lm loss: 3.334098E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.224 | TFLOPs: 71.17 | +[default7]: iteration 1905/ 3814 | consumed samples: 975360 | consumed tokens: 1997537280 | elapsed time per iteration (s): 5.43 | learning rate: 6.237E-04 | global batch size: 512 | lm loss: 3.367708E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.243 | TFLOPs: 71.18 | +[default7]: iteration 1906/ 3814 | consumed samples: 975872 | consumed tokens: 1998585856 | elapsed time per iteration (s): 5.43 | learning rate: 6.233E-04 | global batch size: 512 | lm loss: 3.331143E+00 | loss scale: 32768.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.205 | TFLOPs: 71.15 | +[default7]: iteration 1907/ 3814 | consumed samples: 976384 | consumed tokens: 1999634432 | elapsed time per iteration (s): 5.43 | learning rate: 6.229E-04 | global batch size: 512 | lm loss: 3.337265E+00 | loss scale: 32768.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.348 | TFLOPs: 71.26 | +[default7]: iteration 1908/ 3814 | consumed samples: 976896 | consumed tokens: 2000683008 | elapsed time per iteration (s): 5.44 | learning rate: 6.225E-04 | global batch size: 512 | lm loss: 3.381296E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.165 | TFLOPs: 71.12 | +[default7]: iteration 1909/ 3814 | consumed samples: 977408 | consumed tokens: 2001731584 | elapsed time per iteration (s): 5.42 | learning rate: 6.221E-04 | global batch size: 512 | lm loss: 3.353775E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 1910/ 3814 | consumed samples: 977920 | consumed tokens: 2002780160 | elapsed time per iteration (s): 5.42 | learning rate: 6.217E-04 | global batch size: 512 | lm loss: 3.344316E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.431 | TFLOPs: 71.32 | +[default7]: iteration 1911/ 3814 | consumed samples: 978432 | consumed tokens: 2003828736 | elapsed time per iteration (s): 5.43 | learning rate: 6.213E-04 | global batch size: 512 | lm loss: 3.318454E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.265 | TFLOPs: 71.20 | +[default7]: iteration 1912/ 3814 | consumed samples: 978944 | consumed tokens: 2004877312 | elapsed time per iteration (s): 5.41 | learning rate: 6.209E-04 | global batch size: 512 | lm loss: 3.335335E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.669 | TFLOPs: 71.50 | +[default7]: iteration 1913/ 3814 | consumed samples: 979456 | consumed tokens: 2005925888 | elapsed time per iteration (s): 5.42 | learning rate: 6.205E-04 | global batch size: 512 | lm loss: 3.331695E+00 | loss scale: 32768.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.476 | TFLOPs: 71.36 | +[default7]: iteration 1914/ 3814 | consumed samples: 979968 | consumed tokens: 2006974464 | elapsed time per iteration (s): 5.41 | learning rate: 6.201E-04 | global batch size: 512 | lm loss: 3.343078E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.559 | TFLOPs: 71.42 | +[default7]: iteration 1915/ 3814 | consumed samples: 980480 | consumed tokens: 2008023040 | elapsed time per iteration (s): 5.42 | learning rate: 6.197E-04 | global batch size: 512 | lm loss: 3.319813E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.475 | TFLOPs: 71.36 | +[default7]: iteration 1916/ 3814 | consumed samples: 980992 | consumed tokens: 2009071616 | elapsed time per iteration (s): 5.44 | learning rate: 6.193E-04 | global batch size: 512 | lm loss: 3.308728E+00 | loss scale: 32768.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.156 | TFLOPs: 71.12 | +[default7]: iteration 1917/ 3814 | consumed samples: 981504 | consumed tokens: 2010120192 | elapsed time per iteration (s): 5.41 | learning rate: 6.189E-04 | global batch size: 512 | lm loss: 3.349975E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.685 | TFLOPs: 71.52 | +[default7]: iteration 1918/ 3814 | consumed samples: 982016 | consumed tokens: 2011168768 | elapsed time per iteration (s): 5.44 | learning rate: 6.185E-04 | global batch size: 512 | lm loss: 3.315597E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.192 | TFLOPs: 71.14 | +[default7]: iteration 1919/ 3814 | consumed samples: 982528 | consumed tokens: 2012217344 | elapsed time per iteration (s): 5.40 | learning rate: 6.181E-04 | global batch size: 512 | lm loss: 3.285841E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.761 | TFLOPs: 71.57 | +[default7]: iteration 1920/ 3814 | consumed samples: 983040 | consumed tokens: 2013265920 | elapsed time per iteration (s): 5.42 | learning rate: 6.177E-04 | global batch size: 512 | lm loss: 3.331810E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 1921/ 3814 | consumed samples: 983552 | consumed tokens: 2014314496 | elapsed time per iteration (s): 5.42 | learning rate: 6.173E-04 | global batch size: 512 | lm loss: 3.331361E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.508 | TFLOPs: 71.38 | +[default7]: iteration 1922/ 3814 | consumed samples: 984064 | consumed tokens: 2015363072 | elapsed time per iteration (s): 5.43 | learning rate: 6.169E-04 | global batch size: 512 | lm loss: 3.339599E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.311 | TFLOPs: 71.23 | +[default7]: iteration 1923/ 3814 | consumed samples: 984576 | consumed tokens: 2016411648 | elapsed time per iteration (s): 5.42 | learning rate: 6.165E-04 | global batch size: 512 | lm loss: 3.314862E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.392 | TFLOPs: 71.29 | +[default7]: iteration 1924/ 3814 | consumed samples: 985088 | consumed tokens: 2017460224 | elapsed time per iteration (s): 5.42 | learning rate: 6.161E-04 | global batch size: 512 | lm loss: 3.365015E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.501 | TFLOPs: 71.38 | +[default7]: iteration 1925/ 3814 | consumed samples: 985600 | consumed tokens: 2018508800 | elapsed time per iteration (s): 5.41 | learning rate: 6.157E-04 | global batch size: 512 | lm loss: 3.347093E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 1926/ 3814 | consumed samples: 986112 | consumed tokens: 2019557376 | elapsed time per iteration (s): 5.44 | learning rate: 6.153E-04 | global batch size: 512 | lm loss: 3.333243E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.169 | TFLOPs: 71.13 | +[default7]: iteration 1927/ 3814 | consumed samples: 986624 | consumed tokens: 2020605952 | elapsed time per iteration (s): 5.44 | learning rate: 6.149E-04 | global batch size: 512 | lm loss: 3.328722E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.100 | TFLOPs: 71.07 | +[default7]: iteration 1928/ 3814 | consumed samples: 987136 | consumed tokens: 2021654528 | elapsed time per iteration (s): 5.43 | learning rate: 6.145E-04 | global batch size: 512 | lm loss: 3.359632E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.272 | TFLOPs: 71.20 | +[default7]: iteration 1929/ 3814 | consumed samples: 987648 | consumed tokens: 2022703104 | elapsed time per iteration (s): 5.43 | learning rate: 6.141E-04 | global batch size: 512 | lm loss: 3.316874E+00 | loss scale: 32768.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.365 | TFLOPs: 71.28 | +[default7]: iteration 1930/ 3814 | consumed samples: 988160 | consumed tokens: 2023751680 | elapsed time per iteration (s): 5.41 | learning rate: 6.136E-04 | global batch size: 512 | lm loss: 3.331652E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.722 | TFLOPs: 71.54 | +[default7]: iteration 1931/ 3814 | consumed samples: 988672 | consumed tokens: 2024800256 | elapsed time per iteration (s): 5.43 | learning rate: 6.132E-04 | global batch size: 512 | lm loss: 3.297551E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.345 | TFLOPs: 71.26 | +[default7]: iteration 1932/ 3814 | consumed samples: 989184 | consumed tokens: 2025848832 | elapsed time per iteration (s): 5.43 | learning rate: 6.128E-04 | global batch size: 512 | lm loss: 3.332545E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.377 | TFLOPs: 71.28 | +[default7]: iteration 1933/ 3814 | consumed samples: 989696 | consumed tokens: 2026897408 | elapsed time per iteration (s): 5.40 | learning rate: 6.124E-04 | global batch size: 512 | lm loss: 3.387067E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.749 | TFLOPs: 71.57 | +[default7]: iteration 1934/ 3814 | consumed samples: 990208 | consumed tokens: 2027945984 | elapsed time per iteration (s): 5.42 | learning rate: 6.120E-04 | global batch size: 512 | lm loss: 3.318337E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.509 | TFLOPs: 71.38 | +[default7]: iteration 1935/ 3814 | consumed samples: 990720 | consumed tokens: 2028994560 | elapsed time per iteration (s): 5.41 | learning rate: 6.116E-04 | global batch size: 512 | lm loss: 3.317055E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 1936/ 3814 | consumed samples: 991232 | consumed tokens: 2030043136 | elapsed time per iteration (s): 5.42 | learning rate: 6.112E-04 | global batch size: 512 | lm loss: 3.346162E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.455 | TFLOPs: 71.34 | +[default7]: iteration 1937/ 3814 | consumed samples: 991744 | consumed tokens: 2031091712 | elapsed time per iteration (s): 5.42 | learning rate: 6.108E-04 | global batch size: 512 | lm loss: 3.315740E+00 | loss scale: 32768.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.420 | TFLOPs: 71.32 | +[default7]: iteration 1938/ 3814 | consumed samples: 992256 | consumed tokens: 2032140288 | elapsed time per iteration (s): 5.41 | learning rate: 6.104E-04 | global batch size: 512 | lm loss: 3.350221E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.577 | TFLOPs: 71.43 | +[default7]: iteration 1939/ 3814 | consumed samples: 992768 | consumed tokens: 2033188864 | elapsed time per iteration (s): 5.41 | learning rate: 6.100E-04 | global batch size: 512 | lm loss: 3.324229E+00 | loss scale: 32768.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.679 | TFLOPs: 71.51 | +[default7]: iteration 1940/ 3814 | consumed samples: 993280 | consumed tokens: 2034237440 | elapsed time per iteration (s): 5.42 | learning rate: 6.096E-04 | global batch size: 512 | lm loss: 3.320324E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.508 | TFLOPs: 71.38 | +[default7]: iteration 1941/ 3814 | consumed samples: 993792 | consumed tokens: 2035286016 | elapsed time per iteration (s): 5.44 | learning rate: 6.092E-04 | global batch size: 512 | lm loss: 3.305131E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.164 | TFLOPs: 71.12 | +[default7]: iteration 1942/ 3814 | consumed samples: 994304 | consumed tokens: 2036334592 | elapsed time per iteration (s): 5.40 | learning rate: 6.088E-04 | global batch size: 512 | lm loss: 3.349639E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.764 | TFLOPs: 71.58 | +[default7]: iteration 1943/ 3814 | consumed samples: 994816 | consumed tokens: 2037383168 | elapsed time per iteration (s): 5.41 | learning rate: 6.084E-04 | global batch size: 512 | lm loss: 3.333625E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 1944/ 3814 | consumed samples: 995328 | consumed tokens: 2038431744 | elapsed time per iteration (s): 5.43 | learning rate: 6.080E-04 | global batch size: 512 | lm loss: 3.368329E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.289 | TFLOPs: 71.22 | +[default7]: iteration 1945/ 3814 | consumed samples: 995840 | consumed tokens: 2039480320 | elapsed time per iteration (s): 5.40 | learning rate: 6.076E-04 | global batch size: 512 | lm loss: 3.352720E+00 | loss scale: 32768.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.833 | TFLOPs: 71.63 | +[default7]: iteration 1946/ 3814 | consumed samples: 996352 | consumed tokens: 2040528896 | elapsed time per iteration (s): 5.40 | learning rate: 6.072E-04 | global batch size: 512 | lm loss: 3.313304E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.807 | TFLOPs: 71.61 | +[default7]: iteration 1947/ 3814 | consumed samples: 996864 | consumed tokens: 2041577472 | elapsed time per iteration (s): 5.42 | learning rate: 6.068E-04 | global batch size: 512 | lm loss: 3.320206E+00 | loss scale: 32768.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default7]: iteration 1948/ 3814 | consumed samples: 997376 | consumed tokens: 2042626048 | elapsed time per iteration (s): 5.41 | learning rate: 6.064E-04 | global batch size: 512 | lm loss: 3.324196E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.640 | TFLOPs: 71.48 | +[default7]: iteration 1949/ 3814 | consumed samples: 997888 | consumed tokens: 2043674624 | elapsed time per iteration (s): 5.41 | learning rate: 6.059E-04 | global batch size: 512 | lm loss: 3.337895E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.663 | TFLOPs: 71.50 | +[default7]: iteration 1950/ 3814 | consumed samples: 998400 | consumed tokens: 2044723200 | elapsed time per iteration (s): 5.43 | learning rate: 6.055E-04 | global batch size: 512 | lm loss: 3.313039E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.334 | TFLOPs: 71.25 | +[default7]: iteration 1951/ 3814 | consumed samples: 998912 | consumed tokens: 2045771776 | elapsed time per iteration (s): 5.44 | learning rate: 6.051E-04 | global batch size: 512 | lm loss: 3.317722E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.195 | TFLOPs: 71.15 | +[default7]: iteration 1952/ 3814 | consumed samples: 999424 | consumed tokens: 2046820352 | elapsed time per iteration (s): 5.42 | learning rate: 6.047E-04 | global batch size: 512 | lm loss: 3.337728E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.428 | TFLOPs: 71.32 | +[default7]: iteration 1953/ 3814 | consumed samples: 999936 | consumed tokens: 2047868928 | elapsed time per iteration (s): 5.42 | learning rate: 6.043E-04 | global batch size: 512 | lm loss: 3.324855E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.470 | TFLOPs: 71.35 | +[default7]: iteration 1954/ 3814 | consumed samples: 1000448 | consumed tokens: 2048917504 | elapsed time per iteration (s): 5.42 | learning rate: 6.039E-04 | global batch size: 512 | lm loss: 3.321470E+00 | loss scale: 32768.0 | grad norm: 0.086 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.534 | TFLOPs: 71.40 | +[default7]: iteration 1955/ 3814 | consumed samples: 1000960 | consumed tokens: 2049966080 | elapsed time per iteration (s): 5.43 | learning rate: 6.035E-04 | global batch size: 512 | lm loss: 3.313197E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.258 | TFLOPs: 71.19 | +[default7]: iteration 1956/ 3814 | consumed samples: 1001472 | consumed tokens: 2051014656 | elapsed time per iteration (s): 5.42 | learning rate: 6.031E-04 | global batch size: 512 | lm loss: 3.322524E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.437 | TFLOPs: 71.33 | +[default7]: iteration 1957/ 3814 | consumed samples: 1001984 | consumed tokens: 2052063232 | elapsed time per iteration (s): 5.43 | learning rate: 6.027E-04 | global batch size: 512 | lm loss: 3.331998E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.366 | TFLOPs: 71.28 | +[default7]: iteration 1958/ 3814 | consumed samples: 1002496 | consumed tokens: 2053111808 | elapsed time per iteration (s): 5.42 | learning rate: 6.023E-04 | global batch size: 512 | lm loss: 3.307264E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.469 | TFLOPs: 71.35 | +[default7]: iteration 1959/ 3814 | consumed samples: 1003008 | consumed tokens: 2054160384 | elapsed time per iteration (s): 5.43 | learning rate: 6.019E-04 | global batch size: 512 | lm loss: 3.338484E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.372 | TFLOPs: 71.28 | +[default7]: iteration 1960/ 3814 | consumed samples: 1003520 | consumed tokens: 2055208960 | elapsed time per iteration (s): 5.42 | learning rate: 6.015E-04 | global batch size: 512 | lm loss: 3.333678E+00 | loss scale: 32768.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.471 | TFLOPs: 71.35 | +[default7]: iteration 1961/ 3814 | consumed samples: 1004032 | consumed tokens: 2056257536 | elapsed time per iteration (s): 5.42 | learning rate: 6.011E-04 | global batch size: 512 | lm loss: 3.332534E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.548 | TFLOPs: 71.41 | +[default7]: iteration 1962/ 3814 | consumed samples: 1004544 | consumed tokens: 2057306112 | elapsed time per iteration (s): 5.40 | learning rate: 6.007E-04 | global batch size: 512 | lm loss: 3.322561E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.825 | TFLOPs: 71.62 | +[default7]: iteration 1963/ 3814 | consumed samples: 1005056 | consumed tokens: 2058354688 | elapsed time per iteration (s): 5.43 | learning rate: 6.003E-04 | global batch size: 512 | lm loss: 3.296392E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.236 | TFLOPs: 71.18 | +[default7]: iteration 1964/ 3814 | consumed samples: 1005568 | consumed tokens: 2059403264 | elapsed time per iteration (s): 5.42 | learning rate: 5.999E-04 | global batch size: 512 | lm loss: 3.307299E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 1965/ 3814 | consumed samples: 1006080 | consumed tokens: 2060451840 | elapsed time per iteration (s): 5.43 | learning rate: 5.994E-04 | global batch size: 512 | lm loss: 3.335832E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.306 | TFLOPs: 71.23 | +[default7]: iteration 1966/ 3814 | consumed samples: 1006592 | consumed tokens: 2061500416 | elapsed time per iteration (s): 5.44 | learning rate: 5.990E-04 | global batch size: 512 | lm loss: 3.274970E+00 | loss scale: 32768.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.059 | TFLOPs: 71.04 | +[default7]: iteration 1967/ 3814 | consumed samples: 1007104 | consumed tokens: 2062548992 | elapsed time per iteration (s): 5.43 | learning rate: 5.986E-04 | global batch size: 512 | lm loss: 3.344838E+00 | loss scale: 32768.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.284 | TFLOPs: 71.21 | +[default7]: iteration 1968/ 3814 | consumed samples: 1007616 | consumed tokens: 2063597568 | elapsed time per iteration (s): 5.44 | learning rate: 5.982E-04 | global batch size: 512 | lm loss: 3.305783E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.129 | TFLOPs: 71.10 | +[default7]: iteration 1969/ 3814 | consumed samples: 1008128 | consumed tokens: 2064646144 | elapsed time per iteration (s): 5.42 | learning rate: 5.978E-04 | global batch size: 512 | lm loss: 3.290943E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.382 | TFLOPs: 71.29 | +[default7]: iteration 1970/ 3814 | consumed samples: 1008640 | consumed tokens: 2065694720 | elapsed time per iteration (s): 5.43 | learning rate: 5.974E-04 | global batch size: 512 | lm loss: 3.317065E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.316 | TFLOPs: 71.24 | +[default7]: iteration 1971/ 3814 | consumed samples: 1009152 | consumed tokens: 2066743296 | elapsed time per iteration (s): 5.42 | learning rate: 5.970E-04 | global batch size: 512 | lm loss: 3.306813E+00 | loss scale: 32768.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.504 | TFLOPs: 71.38 | +[default7]: iteration 1972/ 3814 | consumed samples: 1009664 | consumed tokens: 2067791872 | elapsed time per iteration (s): 5.43 | learning rate: 5.966E-04 | global batch size: 512 | lm loss: 3.291399E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.244 | TFLOPs: 71.18 | +[default7]: iteration 1973/ 3814 | consumed samples: 1010176 | consumed tokens: 2068840448 | elapsed time per iteration (s): 5.43 | learning rate: 5.962E-04 | global batch size: 512 | lm loss: 3.318072E+00 | loss scale: 32768.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.217 | TFLOPs: 71.16 | +[default7]: iteration 1974/ 3814 | consumed samples: 1010688 | consumed tokens: 2069889024 | elapsed time per iteration (s): 5.43 | learning rate: 5.958E-04 | global batch size: 512 | lm loss: 3.363374E+00 | loss scale: 32768.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.376 | TFLOPs: 71.28 | +[default7]: iteration 1975/ 3814 | consumed samples: 1011200 | consumed tokens: 2070937600 | elapsed time per iteration (s): 5.44 | learning rate: 5.954E-04 | global batch size: 512 | lm loss: 3.315763E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.138 | TFLOPs: 71.10 | +[default7]: iteration 1976/ 3814 | consumed samples: 1011712 | consumed tokens: 2071986176 | elapsed time per iteration (s): 5.43 | learning rate: 5.950E-04 | global batch size: 512 | lm loss: 3.310376E+00 | loss scale: 32768.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.260 | TFLOPs: 71.20 | +[default7]: iteration 1977/ 3814 | consumed samples: 1012224 | consumed tokens: 2073034752 | elapsed time per iteration (s): 5.43 | learning rate: 5.946E-04 | global batch size: 512 | lm loss: 3.319134E+00 | loss scale: 32768.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.354 | TFLOPs: 71.27 | +[default7]: iteration 1978/ 3814 | consumed samples: 1012736 | consumed tokens: 2074083328 | elapsed time per iteration (s): 5.44 | learning rate: 5.942E-04 | global batch size: 512 | lm loss: 3.305088E+00 | loss scale: 32768.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.148 | TFLOPs: 71.11 | +[default7]: iteration 1979/ 3814 | consumed samples: 1013248 | consumed tokens: 2075131904 | elapsed time per iteration (s): 5.42 | learning rate: 5.938E-04 | global batch size: 512 | lm loss: 3.331315E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.440 | TFLOPs: 71.33 | +[default7]: iteration 1980/ 3814 | consumed samples: 1013760 | consumed tokens: 2076180480 | elapsed time per iteration (s): 5.42 | learning rate: 5.933E-04 | global batch size: 512 | lm loss: 3.298454E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.433 | TFLOPs: 71.33 | +[default7]: iteration 1981/ 3814 | consumed samples: 1014272 | consumed tokens: 2077229056 | elapsed time per iteration (s): 5.42 | learning rate: 5.929E-04 | global batch size: 512 | lm loss: 3.276535E+00 | loss scale: 32768.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.528 | TFLOPs: 71.40 | +[default7]: iteration 1982/ 3814 | consumed samples: 1014784 | consumed tokens: 2078277632 | elapsed time per iteration (s): 5.41 | learning rate: 5.925E-04 | global batch size: 512 | lm loss: 3.337545E+00 | loss scale: 32768.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.591 | TFLOPs: 71.45 | +[default7]: iteration 1983/ 3814 | consumed samples: 1015296 | consumed tokens: 2079326208 | elapsed time per iteration (s): 5.42 | learning rate: 5.921E-04 | global batch size: 512 | lm loss: 3.331475E+00 | loss scale: 32768.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.400 | TFLOPs: 71.30 | +[default7]: iteration 1984/ 3814 | consumed samples: 1015808 | consumed tokens: 2080374784 | elapsed time per iteration (s): 5.42 | learning rate: 5.917E-04 | global batch size: 512 | lm loss: 3.306669E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.473 | TFLOPs: 71.36 | +[default7]: iteration 1985/ 3814 | consumed samples: 1016320 | consumed tokens: 2081423360 | elapsed time per iteration (s): 5.41 | learning rate: 5.913E-04 | global batch size: 512 | lm loss: 3.323497E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.47 | +[default7]: iteration 1986/ 3814 | consumed samples: 1016832 | consumed tokens: 2082471936 | elapsed time per iteration (s): 5.40 | learning rate: 5.909E-04 | global batch size: 512 | lm loss: 3.302235E+00 | loss scale: 32768.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.882 | TFLOPs: 71.67 | +[default7]: iteration 1987/ 3814 | consumed samples: 1017344 | consumed tokens: 2083520512 | elapsed time per iteration (s): 5.43 | learning rate: 5.905E-04 | global batch size: 512 | lm loss: 3.309082E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.335 | TFLOPs: 71.25 | +[default7]: iteration 1988/ 3814 | consumed samples: 1017856 | consumed tokens: 2084569088 | elapsed time per iteration (s): 5.42 | learning rate: 5.901E-04 | global batch size: 512 | lm loss: 3.307788E+00 | loss scale: 32768.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.411 | TFLOPs: 71.31 | +[default7]: iteration 1989/ 3814 | consumed samples: 1018368 | consumed tokens: 2085617664 | elapsed time per iteration (s): 5.41 | learning rate: 5.897E-04 | global batch size: 512 | lm loss: 3.296378E+00 | loss scale: 32768.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.571 | TFLOPs: 71.43 | +[default7]: iteration 1990/ 3814 | consumed samples: 1018880 | consumed tokens: 2086666240 | elapsed time per iteration (s): 5.43 | learning rate: 5.893E-04 | global batch size: 512 | lm loss: 3.305359E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.376 | TFLOPs: 71.28 | +[default7]: iteration 1991/ 3814 | consumed samples: 1019392 | consumed tokens: 2087714816 | elapsed time per iteration (s): 5.43 | learning rate: 5.889E-04 | global batch size: 512 | lm loss: 3.282961E+00 | loss scale: 32768.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.359 | TFLOPs: 71.27 | +[default7]: iteration 1992/ 3814 | consumed samples: 1019904 | consumed tokens: 2088763392 | elapsed time per iteration (s): 5.41 | learning rate: 5.885E-04 | global batch size: 512 | lm loss: 3.333542E+00 | loss scale: 32768.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.645 | TFLOPs: 71.49 | +[default7]: iteration 1993/ 3814 | consumed samples: 1020416 | consumed tokens: 2089811968 | elapsed time per iteration (s): 5.42 | learning rate: 5.881E-04 | global batch size: 512 | lm loss: 3.328329E+00 | loss scale: 32768.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.455 | TFLOPs: 71.34 | +[default7]: iteration 1994/ 3814 | consumed samples: 1020928 | consumed tokens: 2090860544 | elapsed time per iteration (s): 5.41 | learning rate: 5.876E-04 | global batch size: 512 | lm loss: 3.317206E+00 | loss scale: 32768.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.612 | TFLOPs: 71.46 | +[default7]: iteration 1995/ 3814 | consumed samples: 1021440 | consumed tokens: 2091909120 | elapsed time per iteration (s): 5.43 | learning rate: 5.872E-04 | global batch size: 512 | lm loss: 3.285312E+00 | loss scale: 32768.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.235 | TFLOPs: 71.18 | +[default7]: iteration 1996/ 3814 | consumed samples: 1021952 | consumed tokens: 2092957696 | elapsed time per iteration (s): 5.41 | learning rate: 5.868E-04 | global batch size: 512 | lm loss: 3.294050E+00 | loss scale: 32768.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.628 | TFLOPs: 71.47 | +[default7]: iteration 1997/ 3814 | consumed samples: 1022464 | consumed tokens: 2094006272 | elapsed time per iteration (s): 5.41 | learning rate: 5.864E-04 | global batch size: 512 | lm loss: 3.306518E+00 | loss scale: 32768.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.685 | TFLOPs: 71.52 | +[default7]: iteration 1998/ 3814 | consumed samples: 1022976 | consumed tokens: 2095054848 | elapsed time per iteration (s): 5.41 | learning rate: 5.860E-04 | global batch size: 512 | lm loss: 3.326685E+00 | loss scale: 32768.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.613 | TFLOPs: 71.46 | +[default7]: iteration 1999/ 3814 | consumed samples: 1023488 | consumed tokens: 2096103424 | elapsed time per iteration (s): 5.42 | learning rate: 5.856E-04 | global batch size: 512 | lm loss: 3.292803E+00 | loss scale: 32768.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.540 | TFLOPs: 71.41 | +[default0]:[2023-02-16 09:20:30,672] [INFO] [logging.py:68:log_dist] [Rank 0] step=2000, skipped=0, lr=[0.0005851987016942563, 0.00029259935084712815, 0.005851987016942563, 0.00029259935084712815, 0.0005851987016942563, 0.005851987016942563], mom=[(0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95), (0.9, 0.95)] +[default0]:steps: 2000 loss: 3.3088 iter time (s): 5.299 samples/sec: 96.624 +[default7]: iteration 2000/ 3814 | consumed samples: 1024000 | consumed tokens: 2097152000 | elapsed time per iteration (s): 5.39 | learning rate: 5.852E-04 | global batch size: 512 | lm loss: 3.308783E+00 | loss scale: 65536.0 | grad norm: 0.052 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.924 | TFLOPs: 71.70 | +[default0]:saving checkpoint at iteration 2000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 09:20:31,785] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2000 is begin to save! +[default0]:[2023-02-16 09:20:31,789] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_01-model_00-model_states.pt... +[default7]:------------------------------------------------------------------------------------------ +[default7]:valid loss at iteration 2000 | lm loss value: 3.298677E+00 | lm loss PPL: 2.707679E+01 | +[default7]:------------------------------------------------------------------------------------------ +[default1]:[2023-02-16 09:20:31,789] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_01-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:31,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:31,949] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:31,960] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:31,961] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,016] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,017] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,031] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_05-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,098] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_06-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,083] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,083] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,140] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,141] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,157] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,158] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,199] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,200] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,258] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,259] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,217] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,218] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,278] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,278] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,318] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,319] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_10-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,338] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,339] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,399] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,400] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,379] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,380] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_11-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,438] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,463] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,464] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,496] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,497] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_13-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,555] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,556] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,527] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,527] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,587] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,614] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,649] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,713] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,713] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,676] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,677] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_16-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,742] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,781] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,781] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,800] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,801] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_18-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:32,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,858] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_19-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,842] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:32,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,910] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,923] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:32,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:32,971] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:32,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:32,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,039] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,039] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,033] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_21-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:33,096] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,097] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,101] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,102] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,159] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,163] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,163] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,224] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,225] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,222] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_24-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:33,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,284] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,284] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,285] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,364] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,364] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,365] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_27-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,447] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,448] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,450] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,450] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:33,513] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,514] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,505] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,506] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,574] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,574] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,565] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,566] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_30-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,629] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,629] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,639] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:33,701] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,692] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,693] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,757] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,767] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,767] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,818] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,819] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 09:20:33,830] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,831] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_33-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:33,898] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,899] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 09:20:33,884] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,884] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,942] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,943] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 09:20:33,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 09:20:33,944] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_01_model_states.pt +[default1]:[2023-02-16 09:20:33,945] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 09:20:33,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_01_model_states.pt. +[default0]:[2023-02-16 09:20:33,959] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:33,959] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 09:20:34,011] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:34,011] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_37-model_00-model_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:20:34,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 09:20:34,013] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_00_model_states.pt +[default0]:[2023-02-16 09:20:34,013] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 09:20:34,015] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/mp_rank_00_model_states.pt. +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:20:34,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:20:34,536] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,536] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,536] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,602] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,602] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,602] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,557] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,557] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,557] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,618] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,618] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,618] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,634] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,634] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,634] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,673] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,673] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,673] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,667] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,667] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,667] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,709] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,709] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,709] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,684] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,684] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,684] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,700] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,701] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,681] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,681] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,681] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,723] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,723] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,723] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,699] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,699] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,700] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,719] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,719] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,727] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,727] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,746] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,746] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,746] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,728] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,728] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,753] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,753] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,716] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,716] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,716] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,747] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,747] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,747] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,722] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,722] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,722] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,685] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,685] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,685] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,728] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,728] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,759] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,759] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,759] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,729] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,729] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,744] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,744] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,744] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,777] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,777] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,724] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,725] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,729] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,729] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,729] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,695] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,695] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,695] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,726] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,726] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,726] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,732] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,732] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,728] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,729] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,776] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,776] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,785] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,785] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,785] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,739] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,739] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,745] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,745] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,745] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,766] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,766] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,757] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,757] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,778] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,778] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,770] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,770] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,770] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,779] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,801] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,801] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,822] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,822] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,822] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,775] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,775] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,776] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,776] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,776] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,750] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,750] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,772] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,772] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,754] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,754] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,754] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,835] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,835] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,782] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,782] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:34,780] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:34,780] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:34,780] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,814] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,814] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,814] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default3]:[2023-02-16 09:20:34,825] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:20:34,825] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:20:34,825] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:[2023-02-16 09:20:34,827] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:34,828] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:34,828] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,824] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,824] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]:[2023-02-16 09:20:34,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:20:34,907] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:20:34,907] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default5]:[2023-02-16 09:20:34,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:20:34,853] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:20:34,853] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default4]:[2023-02-16 09:20:34,839] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:20:34,839] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:20:34,839] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default1]:[2023-02-16 09:20:34,910] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:20:34,910] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:20:34,910] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default2]:[2023-02-16 09:20:34,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:20:34,908] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:20:34,908] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default6]:[2023-02-16 09:20:35,021] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:20:35,021] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:20:35,021] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default0]: successfully saved checkpoint at iteration 2000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]:[2023-02-16 09:20:35,077] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:20:35,077] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2000/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:20:35,077] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2000 is ready now! +[default7]:time (ms) | save-checkpoint: 3298.76 +[default7]: iteration 2001/ 3814 | consumed samples: 1024512 | consumed tokens: 2098200576 | elapsed time per iteration (s): 9.73 | learning rate: 5.848E-04 | global batch size: 512 | lm loss: 3.317046E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 52.638 | TFLOPs: 39.76 | +[default7]: iteration 2002/ 3814 | consumed samples: 1025024 | consumed tokens: 2099249152 | elapsed time per iteration (s): 5.42 | learning rate: 5.844E-04 | global batch size: 512 | lm loss: 3.324121E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.396 | TFLOPs: 71.30 | +[default7]: iteration 2003/ 3814 | consumed samples: 1025536 | consumed tokens: 2100297728 | elapsed time per iteration (s): 5.47 | learning rate: 5.840E-04 | global batch size: 512 | lm loss: 3.304425E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.671 | TFLOPs: 70.75 | +[default7]: iteration 2004/ 3814 | consumed samples: 1026048 | consumed tokens: 2101346304 | elapsed time per iteration (s): 5.41 | learning rate: 5.836E-04 | global batch size: 512 | lm loss: 3.317958E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.620 | TFLOPs: 71.47 | +[default7]: iteration 2005/ 3814 | consumed samples: 1026560 | consumed tokens: 2102394880 | elapsed time per iteration (s): 5.41 | learning rate: 5.832E-04 | global batch size: 512 | lm loss: 3.314780E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 2006/ 3814 | consumed samples: 1027072 | consumed tokens: 2103443456 | elapsed time per iteration (s): 5.41 | learning rate: 5.828E-04 | global batch size: 512 | lm loss: 3.303081E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.713 | TFLOPs: 71.54 | +[default7]: iteration 2007/ 3814 | consumed samples: 1027584 | consumed tokens: 2104492032 | elapsed time per iteration (s): 5.41 | learning rate: 5.823E-04 | global batch size: 512 | lm loss: 3.300009E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 2008/ 3814 | consumed samples: 1028096 | consumed tokens: 2105540608 | elapsed time per iteration (s): 5.42 | learning rate: 5.819E-04 | global batch size: 512 | lm loss: 3.314317E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 2009/ 3814 | consumed samples: 1028608 | consumed tokens: 2106589184 | elapsed time per iteration (s): 5.42 | learning rate: 5.815E-04 | global batch size: 512 | lm loss: 3.288716E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.384 | TFLOPs: 71.29 | +[default7]: iteration 2010/ 3814 | consumed samples: 1029120 | consumed tokens: 2107637760 | elapsed time per iteration (s): 5.42 | learning rate: 5.811E-04 | global batch size: 512 | lm loss: 3.323458E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.400 | TFLOPs: 71.30 | +[default7]: iteration 2011/ 3814 | consumed samples: 1029632 | consumed tokens: 2108686336 | elapsed time per iteration (s): 5.43 | learning rate: 5.807E-04 | global batch size: 512 | lm loss: 3.330249E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.293 | TFLOPs: 71.22 | +[default7]: iteration 2012/ 3814 | consumed samples: 1030144 | consumed tokens: 2109734912 | elapsed time per iteration (s): 5.43 | learning rate: 5.803E-04 | global batch size: 512 | lm loss: 3.321167E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.244 | TFLOPs: 71.18 | +[default7]: iteration 2013/ 3814 | consumed samples: 1030656 | consumed tokens: 2110783488 | elapsed time per iteration (s): 5.42 | learning rate: 5.799E-04 | global batch size: 512 | lm loss: 3.305259E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.435 | TFLOPs: 71.33 | +[default7]: iteration 2014/ 3814 | consumed samples: 1031168 | consumed tokens: 2111832064 | elapsed time per iteration (s): 5.43 | learning rate: 5.795E-04 | global batch size: 512 | lm loss: 3.311145E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.322 | TFLOPs: 71.24 | +[default7]: iteration 2015/ 3814 | consumed samples: 1031680 | consumed tokens: 2112880640 | elapsed time per iteration (s): 5.42 | learning rate: 5.791E-04 | global batch size: 512 | lm loss: 3.337508E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.456 | TFLOPs: 71.34 | +[default7]: iteration 2016/ 3814 | consumed samples: 1032192 | consumed tokens: 2113929216 | elapsed time per iteration (s): 5.43 | learning rate: 5.787E-04 | global batch size: 512 | lm loss: 3.326778E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.359 | TFLOPs: 71.27 | +[default7]: iteration 2017/ 3814 | consumed samples: 1032704 | consumed tokens: 2114977792 | elapsed time per iteration (s): 5.43 | learning rate: 5.783E-04 | global batch size: 512 | lm loss: 3.328387E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.249 | TFLOPs: 71.19 | +[default7]: iteration 2018/ 3814 | consumed samples: 1033216 | consumed tokens: 2116026368 | elapsed time per iteration (s): 5.41 | learning rate: 5.779E-04 | global batch size: 512 | lm loss: 3.303737E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.555 | TFLOPs: 71.42 | +[default7]: iteration 2019/ 3814 | consumed samples: 1033728 | consumed tokens: 2117074944 | elapsed time per iteration (s): 5.42 | learning rate: 5.774E-04 | global batch size: 512 | lm loss: 3.298887E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 2020/ 3814 | consumed samples: 1034240 | consumed tokens: 2118123520 | elapsed time per iteration (s): 5.41 | learning rate: 5.770E-04 | global batch size: 512 | lm loss: 3.284923E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.726 | TFLOPs: 71.55 | +[default7]: iteration 2021/ 3814 | consumed samples: 1034752 | consumed tokens: 2119172096 | elapsed time per iteration (s): 5.43 | learning rate: 5.766E-04 | global batch size: 512 | lm loss: 3.282894E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.324 | TFLOPs: 71.24 | +[default7]: iteration 2022/ 3814 | consumed samples: 1035264 | consumed tokens: 2120220672 | elapsed time per iteration (s): 5.41 | learning rate: 5.762E-04 | global batch size: 512 | lm loss: 3.327017E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.684 | TFLOPs: 71.52 | +[default7]: iteration 2023/ 3814 | consumed samples: 1035776 | consumed tokens: 2121269248 | elapsed time per iteration (s): 5.44 | learning rate: 5.758E-04 | global batch size: 512 | lm loss: 3.262494E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.189 | TFLOPs: 71.14 | +[default7]: iteration 2024/ 3814 | consumed samples: 1036288 | consumed tokens: 2122317824 | elapsed time per iteration (s): 5.41 | learning rate: 5.754E-04 | global batch size: 512 | lm loss: 3.301629E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default7]: iteration 2025/ 3814 | consumed samples: 1036800 | consumed tokens: 2123366400 | elapsed time per iteration (s): 5.44 | learning rate: 5.750E-04 | global batch size: 512 | lm loss: 3.318487E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.192 | TFLOPs: 71.14 | +[default7]: iteration 2026/ 3814 | consumed samples: 1037312 | consumed tokens: 2124414976 | elapsed time per iteration (s): 5.41 | learning rate: 5.746E-04 | global batch size: 512 | lm loss: 3.295015E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.584 | TFLOPs: 71.44 | +[default7]: iteration 2027/ 3814 | consumed samples: 1037824 | consumed tokens: 2125463552 | elapsed time per iteration (s): 5.43 | learning rate: 5.742E-04 | global batch size: 512 | lm loss: 3.278988E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.338 | TFLOPs: 71.25 | +[default7]: iteration 2028/ 3814 | consumed samples: 1038336 | consumed tokens: 2126512128 | elapsed time per iteration (s): 5.43 | learning rate: 5.738E-04 | global batch size: 512 | lm loss: 3.345369E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.215 | TFLOPs: 71.16 | +[default7]: iteration 2029/ 3814 | consumed samples: 1038848 | consumed tokens: 2127560704 | elapsed time per iteration (s): 5.43 | learning rate: 5.734E-04 | global batch size: 512 | lm loss: 3.291781E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.323 | TFLOPs: 71.24 | +[default7]: iteration 2030/ 3814 | consumed samples: 1039360 | consumed tokens: 2128609280 | elapsed time per iteration (s): 5.42 | learning rate: 5.730E-04 | global batch size: 512 | lm loss: 3.286630E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.441 | TFLOPs: 71.33 | +[default7]: iteration 2031/ 3814 | consumed samples: 1039872 | consumed tokens: 2129657856 | elapsed time per iteration (s): 5.42 | learning rate: 5.725E-04 | global batch size: 512 | lm loss: 3.292575E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.461 | TFLOPs: 71.35 | +[default7]: iteration 2032/ 3814 | consumed samples: 1040384 | consumed tokens: 2130706432 | elapsed time per iteration (s): 5.43 | learning rate: 5.721E-04 | global batch size: 512 | lm loss: 3.315156E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.347 | TFLOPs: 71.26 | +[default7]: iteration 2033/ 3814 | consumed samples: 1040896 | consumed tokens: 2131755008 | elapsed time per iteration (s): 5.42 | learning rate: 5.717E-04 | global batch size: 512 | lm loss: 3.290942E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.433 | TFLOPs: 71.33 | +[default7]: iteration 2034/ 3814 | consumed samples: 1041408 | consumed tokens: 2132803584 | elapsed time per iteration (s): 5.42 | learning rate: 5.713E-04 | global batch size: 512 | lm loss: 3.286292E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.515 | TFLOPs: 71.39 | +[default7]: iteration 2035/ 3814 | consumed samples: 1041920 | consumed tokens: 2133852160 | elapsed time per iteration (s): 5.42 | learning rate: 5.709E-04 | global batch size: 512 | lm loss: 3.315930E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.412 | TFLOPs: 71.31 | +[default7]: iteration 2036/ 3814 | consumed samples: 1042432 | consumed tokens: 2134900736 | elapsed time per iteration (s): 5.42 | learning rate: 5.705E-04 | global batch size: 512 | lm loss: 3.315892E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.542 | TFLOPs: 71.41 | +[default7]: iteration 2037/ 3814 | consumed samples: 1042944 | consumed tokens: 2135949312 | elapsed time per iteration (s): 5.46 | learning rate: 5.701E-04 | global batch size: 512 | lm loss: 3.303986E+00 | loss scale: 65536.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.791 | TFLOPs: 70.84 | +[default7]: iteration 2038/ 3814 | consumed samples: 1043456 | consumed tokens: 2136997888 | elapsed time per iteration (s): 5.43 | learning rate: 5.697E-04 | global batch size: 512 | lm loss: 3.254233E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.353 | TFLOPs: 71.27 | +[default7]: iteration 2039/ 3814 | consumed samples: 1043968 | consumed tokens: 2138046464 | elapsed time per iteration (s): 5.43 | learning rate: 5.693E-04 | global batch size: 512 | lm loss: 3.300475E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.297 | TFLOPs: 71.22 | +[default7]: iteration 2040/ 3814 | consumed samples: 1044480 | consumed tokens: 2139095040 | elapsed time per iteration (s): 5.42 | learning rate: 5.689E-04 | global batch size: 512 | lm loss: 3.323288E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.400 | TFLOPs: 71.30 | +[default7]: iteration 2041/ 3814 | consumed samples: 1044992 | consumed tokens: 2140143616 | elapsed time per iteration (s): 5.42 | learning rate: 5.685E-04 | global batch size: 512 | lm loss: 3.296753E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.529 | TFLOPs: 71.40 | +[default7]: iteration 2042/ 3814 | consumed samples: 1045504 | consumed tokens: 2141192192 | elapsed time per iteration (s): 5.43 | learning rate: 5.681E-04 | global batch size: 512 | lm loss: 3.304849E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.337 | TFLOPs: 71.25 | +[default7]: iteration 2043/ 3814 | consumed samples: 1046016 | consumed tokens: 2142240768 | elapsed time per iteration (s): 5.41 | learning rate: 5.676E-04 | global batch size: 512 | lm loss: 3.301957E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 2044/ 3814 | consumed samples: 1046528 | consumed tokens: 2143289344 | elapsed time per iteration (s): 5.43 | learning rate: 5.672E-04 | global batch size: 512 | lm loss: 3.293361E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.320 | TFLOPs: 71.24 | +[default7]: iteration 2045/ 3814 | consumed samples: 1047040 | consumed tokens: 2144337920 | elapsed time per iteration (s): 5.42 | learning rate: 5.668E-04 | global batch size: 512 | lm loss: 3.310550E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.471 | TFLOPs: 71.36 | +[default7]: iteration 2046/ 3814 | consumed samples: 1047552 | consumed tokens: 2145386496 | elapsed time per iteration (s): 5.41 | learning rate: 5.664E-04 | global batch size: 512 | lm loss: 3.287485E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.698 | TFLOPs: 71.53 | +[default7]: iteration 2047/ 3814 | consumed samples: 1048064 | consumed tokens: 2146435072 | elapsed time per iteration (s): 5.41 | learning rate: 5.660E-04 | global batch size: 512 | lm loss: 3.308741E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 2048/ 3814 | consumed samples: 1048576 | consumed tokens: 2147483648 | elapsed time per iteration (s): 5.41 | learning rate: 5.656E-04 | global batch size: 512 | lm loss: 3.302634E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.673 | TFLOPs: 71.51 | +[default7]: iteration 2049/ 3814 | consumed samples: 1049088 | consumed tokens: 2148532224 | elapsed time per iteration (s): 5.44 | learning rate: 5.652E-04 | global batch size: 512 | lm loss: 3.277282E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.169 | TFLOPs: 71.13 | +[default7]: iteration 2050/ 3814 | consumed samples: 1049600 | consumed tokens: 2149580800 | elapsed time per iteration (s): 5.43 | learning rate: 5.648E-04 | global batch size: 512 | lm loss: 3.311399E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.372 | TFLOPs: 71.28 | +[default7]: iteration 2051/ 3814 | consumed samples: 1050112 | consumed tokens: 2150629376 | elapsed time per iteration (s): 5.42 | learning rate: 5.644E-04 | global batch size: 512 | lm loss: 3.295443E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.429 | TFLOPs: 71.32 | +[default7]: iteration 2052/ 3814 | consumed samples: 1050624 | consumed tokens: 2151677952 | elapsed time per iteration (s): 5.42 | learning rate: 5.640E-04 | global batch size: 512 | lm loss: 3.319851E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 2053/ 3814 | consumed samples: 1051136 | consumed tokens: 2152726528 | elapsed time per iteration (s): 5.41 | learning rate: 5.636E-04 | global batch size: 512 | lm loss: 3.280289E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.627 | TFLOPs: 71.47 | +[default7]: iteration 2054/ 3814 | consumed samples: 1051648 | consumed tokens: 2153775104 | elapsed time per iteration (s): 5.39 | learning rate: 5.632E-04 | global batch size: 512 | lm loss: 3.292400E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.932 | TFLOPs: 71.70 | +[default7]: iteration 2055/ 3814 | consumed samples: 1052160 | consumed tokens: 2154823680 | elapsed time per iteration (s): 5.43 | learning rate: 5.627E-04 | global batch size: 512 | lm loss: 3.306346E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.310 | TFLOPs: 71.23 | +[default7]: iteration 2056/ 3814 | consumed samples: 1052672 | consumed tokens: 2155872256 | elapsed time per iteration (s): 5.40 | learning rate: 5.623E-04 | global batch size: 512 | lm loss: 3.279931E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.812 | TFLOPs: 71.61 | +[default7]: iteration 2057/ 3814 | consumed samples: 1053184 | consumed tokens: 2156920832 | elapsed time per iteration (s): 5.40 | learning rate: 5.619E-04 | global batch size: 512 | lm loss: 3.302739E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.761 | TFLOPs: 71.57 | +[default7]: iteration 2058/ 3814 | consumed samples: 1053696 | consumed tokens: 2157969408 | elapsed time per iteration (s): 5.39 | learning rate: 5.615E-04 | global batch size: 512 | lm loss: 3.301011E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.958 | TFLOPs: 71.72 | +[default7]: iteration 2059/ 3814 | consumed samples: 1054208 | consumed tokens: 2159017984 | elapsed time per iteration (s): 5.40 | learning rate: 5.611E-04 | global batch size: 512 | lm loss: 3.310479E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.777 | TFLOPs: 71.59 | +[default7]: iteration 2060/ 3814 | consumed samples: 1054720 | consumed tokens: 2160066560 | elapsed time per iteration (s): 5.40 | learning rate: 5.607E-04 | global batch size: 512 | lm loss: 3.323296E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.861 | TFLOPs: 71.65 | +[default7]: iteration 2061/ 3814 | consumed samples: 1055232 | consumed tokens: 2161115136 | elapsed time per iteration (s): 5.39 | learning rate: 5.603E-04 | global batch size: 512 | lm loss: 3.266571E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.972 | TFLOPs: 71.73 | +[default7]: iteration 2062/ 3814 | consumed samples: 1055744 | consumed tokens: 2162163712 | elapsed time per iteration (s): 5.41 | learning rate: 5.599E-04 | global batch size: 512 | lm loss: 3.249701E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 2063/ 3814 | consumed samples: 1056256 | consumed tokens: 2163212288 | elapsed time per iteration (s): 5.40 | learning rate: 5.595E-04 | global batch size: 512 | lm loss: 3.277321E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.775 | TFLOPs: 71.58 | +[default7]: iteration 2064/ 3814 | consumed samples: 1056768 | consumed tokens: 2164260864 | elapsed time per iteration (s): 5.41 | learning rate: 5.591E-04 | global batch size: 512 | lm loss: 3.292956E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.666 | TFLOPs: 71.50 | +[default7]: iteration 2065/ 3814 | consumed samples: 1057280 | consumed tokens: 2165309440 | elapsed time per iteration (s): 5.40 | learning rate: 5.587E-04 | global batch size: 512 | lm loss: 3.285674E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.837 | TFLOPs: 71.63 | +[default7]: iteration 2066/ 3814 | consumed samples: 1057792 | consumed tokens: 2166358016 | elapsed time per iteration (s): 5.41 | learning rate: 5.582E-04 | global batch size: 512 | lm loss: 3.302029E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.591 | TFLOPs: 71.45 | +[default7]: iteration 2067/ 3814 | consumed samples: 1058304 | consumed tokens: 2167406592 | elapsed time per iteration (s): 5.40 | learning rate: 5.578E-04 | global batch size: 512 | lm loss: 3.295369E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.860 | TFLOPs: 71.65 | +[default7]: iteration 2068/ 3814 | consumed samples: 1058816 | consumed tokens: 2168455168 | elapsed time per iteration (s): 5.40 | learning rate: 5.574E-04 | global batch size: 512 | lm loss: 3.277787E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.768 | TFLOPs: 71.58 | +[default7]: iteration 2069/ 3814 | consumed samples: 1059328 | consumed tokens: 2169503744 | elapsed time per iteration (s): 5.41 | learning rate: 5.570E-04 | global batch size: 512 | lm loss: 3.291096E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.642 | TFLOPs: 71.48 | +[default7]: iteration 2070/ 3814 | consumed samples: 1059840 | consumed tokens: 2170552320 | elapsed time per iteration (s): 5.42 | learning rate: 5.566E-04 | global batch size: 512 | lm loss: 3.281893E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 2071/ 3814 | consumed samples: 1060352 | consumed tokens: 2171600896 | elapsed time per iteration (s): 5.40 | learning rate: 5.562E-04 | global batch size: 512 | lm loss: 3.318392E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.834 | TFLOPs: 71.63 | +[default7]: iteration 2072/ 3814 | consumed samples: 1060864 | consumed tokens: 2172649472 | elapsed time per iteration (s): 5.42 | learning rate: 5.558E-04 | global batch size: 512 | lm loss: 3.286829E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.494 | TFLOPs: 71.37 | +[default7]: iteration 2073/ 3814 | consumed samples: 1061376 | consumed tokens: 2173698048 | elapsed time per iteration (s): 5.41 | learning rate: 5.554E-04 | global batch size: 512 | lm loss: 3.324224E+00 | loss scale: 65536.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.635 | TFLOPs: 71.48 | +[default7]: iteration 2074/ 3814 | consumed samples: 1061888 | consumed tokens: 2174746624 | elapsed time per iteration (s): 5.41 | learning rate: 5.550E-04 | global batch size: 512 | lm loss: 3.313040E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.725 | TFLOPs: 71.55 | +[default7]: iteration 2075/ 3814 | consumed samples: 1062400 | consumed tokens: 2175795200 | elapsed time per iteration (s): 5.42 | learning rate: 5.546E-04 | global batch size: 512 | lm loss: 3.305075E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 2076/ 3814 | consumed samples: 1062912 | consumed tokens: 2176843776 | elapsed time per iteration (s): 5.42 | learning rate: 5.542E-04 | global batch size: 512 | lm loss: 3.267503E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 2077/ 3814 | consumed samples: 1063424 | consumed tokens: 2177892352 | elapsed time per iteration (s): 5.42 | learning rate: 5.537E-04 | global batch size: 512 | lm loss: 3.296798E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 2078/ 3814 | consumed samples: 1063936 | consumed tokens: 2178940928 | elapsed time per iteration (s): 5.42 | learning rate: 5.533E-04 | global batch size: 512 | lm loss: 3.255067E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.29 | +[default7]: iteration 2079/ 3814 | consumed samples: 1064448 | consumed tokens: 2179989504 | elapsed time per iteration (s): 5.41 | learning rate: 5.529E-04 | global batch size: 512 | lm loss: 3.316163E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.710 | TFLOPs: 71.54 | +[default7]: iteration 2080/ 3814 | consumed samples: 1064960 | consumed tokens: 2181038080 | elapsed time per iteration (s): 5.40 | learning rate: 5.525E-04 | global batch size: 512 | lm loss: 3.317634E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.875 | TFLOPs: 71.66 | +[default7]: iteration 2081/ 3814 | consumed samples: 1065472 | consumed tokens: 2182086656 | elapsed time per iteration (s): 5.41 | learning rate: 5.521E-04 | global batch size: 512 | lm loss: 3.254227E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 2082/ 3814 | consumed samples: 1065984 | consumed tokens: 2183135232 | elapsed time per iteration (s): 5.40 | learning rate: 5.517E-04 | global batch size: 512 | lm loss: 3.291028E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.797 | TFLOPs: 71.60 | +[default7]: iteration 2083/ 3814 | consumed samples: 1066496 | consumed tokens: 2184183808 | elapsed time per iteration (s): 5.40 | learning rate: 5.513E-04 | global batch size: 512 | lm loss: 3.264874E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.64 | +[default7]: iteration 2084/ 3814 | consumed samples: 1067008 | consumed tokens: 2185232384 | elapsed time per iteration (s): 5.43 | learning rate: 5.509E-04 | global batch size: 512 | lm loss: 3.263871E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.267 | TFLOPs: 71.20 | +[default7]: iteration 2085/ 3814 | consumed samples: 1067520 | consumed tokens: 2186280960 | elapsed time per iteration (s): 5.40 | learning rate: 5.505E-04 | global batch size: 512 | lm loss: 3.289559E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.869 | TFLOPs: 71.66 | +[default7]: iteration 2086/ 3814 | consumed samples: 1068032 | consumed tokens: 2187329536 | elapsed time per iteration (s): 5.40 | learning rate: 5.501E-04 | global batch size: 512 | lm loss: 3.273418E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.853 | TFLOPs: 71.64 | +[default7]: iteration 2087/ 3814 | consumed samples: 1068544 | consumed tokens: 2188378112 | elapsed time per iteration (s): 5.41 | learning rate: 5.497E-04 | global batch size: 512 | lm loss: 3.296792E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 2088/ 3814 | consumed samples: 1069056 | consumed tokens: 2189426688 | elapsed time per iteration (s): 5.41 | learning rate: 5.492E-04 | global batch size: 512 | lm loss: 3.284136E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.706 | TFLOPs: 71.53 | +[default7]: iteration 2089/ 3814 | consumed samples: 1069568 | consumed tokens: 2190475264 | elapsed time per iteration (s): 5.41 | learning rate: 5.488E-04 | global batch size: 512 | lm loss: 3.286780E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.621 | TFLOPs: 71.47 | +[default7]: iteration 2090/ 3814 | consumed samples: 1070080 | consumed tokens: 2191523840 | elapsed time per iteration (s): 5.40 | learning rate: 5.484E-04 | global batch size: 512 | lm loss: 3.291673E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.757 | TFLOPs: 71.57 | +[default7]: iteration 2091/ 3814 | consumed samples: 1070592 | consumed tokens: 2192572416 | elapsed time per iteration (s): 5.40 | learning rate: 5.480E-04 | global batch size: 512 | lm loss: 3.292938E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.781 | TFLOPs: 71.59 | +[default7]: iteration 2092/ 3814 | consumed samples: 1071104 | consumed tokens: 2193620992 | elapsed time per iteration (s): 5.42 | learning rate: 5.476E-04 | global batch size: 512 | lm loss: 3.279787E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.532 | TFLOPs: 71.40 | +[default7]: iteration 2093/ 3814 | consumed samples: 1071616 | consumed tokens: 2194669568 | elapsed time per iteration (s): 5.41 | learning rate: 5.472E-04 | global batch size: 512 | lm loss: 3.271140E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 2094/ 3814 | consumed samples: 1072128 | consumed tokens: 2195718144 | elapsed time per iteration (s): 5.40 | learning rate: 5.468E-04 | global batch size: 512 | lm loss: 3.303010E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.851 | TFLOPs: 71.64 | +[default7]: iteration 2095/ 3814 | consumed samples: 1072640 | consumed tokens: 2196766720 | elapsed time per iteration (s): 5.40 | learning rate: 5.464E-04 | global batch size: 512 | lm loss: 3.255980E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.745 | TFLOPs: 71.56 | +[default7]: iteration 2096/ 3814 | consumed samples: 1073152 | consumed tokens: 2197815296 | elapsed time per iteration (s): 5.40 | learning rate: 5.460E-04 | global batch size: 512 | lm loss: 3.257632E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.862 | TFLOPs: 71.65 | +[default7]: iteration 2097/ 3814 | consumed samples: 1073664 | consumed tokens: 2198863872 | elapsed time per iteration (s): 5.41 | learning rate: 5.456E-04 | global batch size: 512 | lm loss: 3.305803E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.48 | +[default7]: iteration 2098/ 3814 | consumed samples: 1074176 | consumed tokens: 2199912448 | elapsed time per iteration (s): 5.40 | learning rate: 5.452E-04 | global batch size: 512 | lm loss: 3.284114E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.844 | TFLOPs: 71.64 | +[default7]: iteration 2099/ 3814 | consumed samples: 1074688 | consumed tokens: 2200961024 | elapsed time per iteration (s): 5.39 | learning rate: 5.448E-04 | global batch size: 512 | lm loss: 3.244901E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.963 | TFLOPs: 71.73 | +[default7]: iteration 2100/ 3814 | consumed samples: 1075200 | consumed tokens: 2202009600 | elapsed time per iteration (s): 5.40 | learning rate: 5.443E-04 | global batch size: 512 | lm loss: 3.266109E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.777 | TFLOPs: 71.59 | +[default7]: iteration 2101/ 3814 | consumed samples: 1075712 | consumed tokens: 2203058176 | elapsed time per iteration (s): 5.40 | learning rate: 5.439E-04 | global batch size: 512 | lm loss: 3.305945E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.753 | TFLOPs: 71.57 | +[default7]: iteration 2102/ 3814 | consumed samples: 1076224 | consumed tokens: 2204106752 | elapsed time per iteration (s): 5.41 | learning rate: 5.435E-04 | global batch size: 512 | lm loss: 3.291470E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.593 | TFLOPs: 71.45 | +[default7]: iteration 2103/ 3814 | consumed samples: 1076736 | consumed tokens: 2205155328 | elapsed time per iteration (s): 5.39 | learning rate: 5.431E-04 | global batch size: 512 | lm loss: 3.288791E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.903 | TFLOPs: 71.68 | +[default7]: iteration 2104/ 3814 | consumed samples: 1077248 | consumed tokens: 2206203904 | elapsed time per iteration (s): 5.39 | learning rate: 5.427E-04 | global batch size: 512 | lm loss: 3.277887E+00 | loss scale: 65536.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.919 | TFLOPs: 71.69 | +[default7]: iteration 2105/ 3814 | consumed samples: 1077760 | consumed tokens: 2207252480 | elapsed time per iteration (s): 5.41 | learning rate: 5.423E-04 | global batch size: 512 | lm loss: 3.259923E+00 | loss scale: 65536.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.600 | TFLOPs: 71.45 | +[default7]: iteration 2106/ 3814 | consumed samples: 1078272 | consumed tokens: 2208301056 | elapsed time per iteration (s): 5.40 | learning rate: 5.419E-04 | global batch size: 512 | lm loss: 3.294288E+00 | loss scale: 65536.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.877 | TFLOPs: 71.66 | +[default7]: iteration 2107/ 3814 | consumed samples: 1078784 | consumed tokens: 2209349632 | elapsed time per iteration (s): 5.39 | learning rate: 5.415E-04 | global batch size: 512 | lm loss: 3.292167E+00 | loss scale: 65536.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.984 | TFLOPs: 71.74 | +[default7]: iteration 2108/ 3814 | consumed samples: 1079296 | consumed tokens: 2210398208 | elapsed time per iteration (s): 5.38 | learning rate: 5.411E-04 | global batch size: 512 | lm loss: 3.264317E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.129 | TFLOPs: 71.85 | +[default7]: iteration 2109/ 3814 | consumed samples: 1079808 | consumed tokens: 2211446784 | elapsed time per iteration (s): 5.39 | learning rate: 5.407E-04 | global batch size: 512 | lm loss: 3.293244E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.952 | TFLOPs: 71.72 | +[default7]: iteration 2110/ 3814 | consumed samples: 1080320 | consumed tokens: 2212495360 | elapsed time per iteration (s): 5.39 | learning rate: 5.403E-04 | global batch size: 512 | lm loss: 3.267266E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.023 | TFLOPs: 71.77 | +[default7]: iteration 2111/ 3814 | consumed samples: 1080832 | consumed tokens: 2213543936 | elapsed time per iteration (s): 5.40 | learning rate: 5.398E-04 | global batch size: 512 | lm loss: 3.280481E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.830 | TFLOPs: 71.63 | +[default7]: iteration 2112/ 3814 | consumed samples: 1081344 | consumed tokens: 2214592512 | elapsed time per iteration (s): 5.39 | learning rate: 5.394E-04 | global batch size: 512 | lm loss: 3.264849E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.995 | TFLOPs: 71.75 | +[default7]: iteration 2113/ 3814 | consumed samples: 1081856 | consumed tokens: 2215641088 | elapsed time per iteration (s): 5.40 | learning rate: 5.390E-04 | global batch size: 512 | lm loss: 3.282069E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.872 | TFLOPs: 71.66 | +[default7]: iteration 2114/ 3814 | consumed samples: 1082368 | consumed tokens: 2216689664 | elapsed time per iteration (s): 5.41 | learning rate: 5.386E-04 | global batch size: 512 | lm loss: 3.289531E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.710 | TFLOPs: 71.54 | +[default7]: iteration 2115/ 3814 | consumed samples: 1082880 | consumed tokens: 2217738240 | elapsed time per iteration (s): 5.39 | learning rate: 5.382E-04 | global batch size: 512 | lm loss: 3.304243E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.990 | TFLOPs: 71.75 | +[default7]: iteration 2116/ 3814 | consumed samples: 1083392 | consumed tokens: 2218786816 | elapsed time per iteration (s): 5.40 | learning rate: 5.378E-04 | global batch size: 512 | lm loss: 3.286122E+00 | loss scale: 65536.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.755 | TFLOPs: 71.57 | +[default7]: iteration 2117/ 3814 | consumed samples: 1083904 | consumed tokens: 2219835392 | elapsed time per iteration (s): 5.39 | learning rate: 5.374E-04 | global batch size: 512 | lm loss: 3.262416E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.037 | TFLOPs: 71.78 | +[default7]: iteration 2118/ 3814 | consumed samples: 1084416 | consumed tokens: 2220883968 | elapsed time per iteration (s): 5.41 | learning rate: 5.370E-04 | global batch size: 512 | lm loss: 3.278454E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2119/ 3814 | consumed samples: 1084928 | consumed tokens: 2221932544 | elapsed time per iteration (s): 5.42 | learning rate: 5.366E-04 | global batch size: 512 | lm loss: 3.298737E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.405 | TFLOPs: 71.30 | +[default7]: iteration 2120/ 3814 | consumed samples: 1085440 | consumed tokens: 2222981120 | elapsed time per iteration (s): 5.39 | learning rate: 5.362E-04 | global batch size: 512 | lm loss: 3.261513E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.909 | TFLOPs: 71.69 | +[default7]: iteration 2121/ 3814 | consumed samples: 1085952 | consumed tokens: 2224029696 | elapsed time per iteration (s): 5.39 | learning rate: 5.358E-04 | global batch size: 512 | lm loss: 3.259652E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.962 | TFLOPs: 71.73 | +[default7]: iteration 2122/ 3814 | consumed samples: 1086464 | consumed tokens: 2225078272 | elapsed time per iteration (s): 5.40 | learning rate: 5.353E-04 | global batch size: 512 | lm loss: 3.250575E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.865 | TFLOPs: 71.65 | +[default7]: iteration 2123/ 3814 | consumed samples: 1086976 | consumed tokens: 2226126848 | elapsed time per iteration (s): 5.41 | learning rate: 5.349E-04 | global batch size: 512 | lm loss: 3.278316E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.571 | TFLOPs: 71.43 | +[default7]: iteration 2124/ 3814 | consumed samples: 1087488 | consumed tokens: 2227175424 | elapsed time per iteration (s): 5.40 | learning rate: 5.345E-04 | global batch size: 512 | lm loss: 3.274937E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.887 | TFLOPs: 71.67 | +[default7]: iteration 2125/ 3814 | consumed samples: 1088000 | consumed tokens: 2228224000 | elapsed time per iteration (s): 5.41 | learning rate: 5.341E-04 | global batch size: 512 | lm loss: 3.285018E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.700 | TFLOPs: 71.53 | +[default7]: iteration 2126/ 3814 | consumed samples: 1088512 | consumed tokens: 2229272576 | elapsed time per iteration (s): 5.40 | learning rate: 5.337E-04 | global batch size: 512 | lm loss: 3.271813E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.741 | TFLOPs: 71.56 | +[default7]: iteration 2127/ 3814 | consumed samples: 1089024 | consumed tokens: 2230321152 | elapsed time per iteration (s): 5.40 | learning rate: 5.333E-04 | global batch size: 512 | lm loss: 3.276792E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.744 | TFLOPs: 71.56 | +[default7]: iteration 2128/ 3814 | consumed samples: 1089536 | consumed tokens: 2231369728 | elapsed time per iteration (s): 5.41 | learning rate: 5.329E-04 | global batch size: 512 | lm loss: 3.271359E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.564 | TFLOPs: 71.43 | +[default7]: iteration 2129/ 3814 | consumed samples: 1090048 | consumed tokens: 2232418304 | elapsed time per iteration (s): 5.38 | learning rate: 5.325E-04 | global batch size: 512 | lm loss: 3.268380E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.106 | TFLOPs: 71.83 | +[default7]: iteration 2130/ 3814 | consumed samples: 1090560 | consumed tokens: 2233466880 | elapsed time per iteration (s): 5.39 | learning rate: 5.321E-04 | global batch size: 512 | lm loss: 3.244395E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.016 | TFLOPs: 71.77 | +[default7]: iteration 2131/ 3814 | consumed samples: 1091072 | consumed tokens: 2234515456 | elapsed time per iteration (s): 5.40 | learning rate: 5.317E-04 | global batch size: 512 | lm loss: 3.291167E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.863 | TFLOPs: 71.65 | +[default7]: iteration 2132/ 3814 | consumed samples: 1091584 | consumed tokens: 2235564032 | elapsed time per iteration (s): 5.40 | learning rate: 5.313E-04 | global batch size: 512 | lm loss: 3.284848E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.887 | TFLOPs: 71.67 | +[default7]: iteration 2133/ 3814 | consumed samples: 1092096 | consumed tokens: 2236612608 | elapsed time per iteration (s): 5.40 | learning rate: 5.309E-04 | global batch size: 512 | lm loss: 3.285958E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.780 | TFLOPs: 71.59 | +[default7]: iteration 2134/ 3814 | consumed samples: 1092608 | consumed tokens: 2237661184 | elapsed time per iteration (s): 5.40 | learning rate: 5.304E-04 | global batch size: 512 | lm loss: 3.264980E+00 | loss scale: 65536.0 | grad norm: 0.087 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.796 | TFLOPs: 71.60 | +[default7]: iteration 2135/ 3814 | consumed samples: 1093120 | consumed tokens: 2238709760 | elapsed time per iteration (s): 5.40 | learning rate: 5.300E-04 | global batch size: 512 | lm loss: 3.257305E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.850 | TFLOPs: 71.64 | +[default7]: iteration 2136/ 3814 | consumed samples: 1093632 | consumed tokens: 2239758336 | elapsed time per iteration (s): 5.40 | learning rate: 5.296E-04 | global batch size: 512 | lm loss: 3.259130E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.848 | TFLOPs: 71.64 | +[default7]: iteration 2137/ 3814 | consumed samples: 1094144 | consumed tokens: 2240806912 | elapsed time per iteration (s): 5.39 | learning rate: 5.292E-04 | global batch size: 512 | lm loss: 3.288483E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.947 | TFLOPs: 71.71 | +[default7]: iteration 2138/ 3814 | consumed samples: 1094656 | consumed tokens: 2241855488 | elapsed time per iteration (s): 5.42 | learning rate: 5.288E-04 | global batch size: 512 | lm loss: 3.276713E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.454 | TFLOPs: 71.34 | +[default7]: iteration 2139/ 3814 | consumed samples: 1095168 | consumed tokens: 2242904064 | elapsed time per iteration (s): 5.39 | learning rate: 5.284E-04 | global batch size: 512 | lm loss: 3.245789E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.949 | TFLOPs: 71.72 | +[default7]: iteration 2140/ 3814 | consumed samples: 1095680 | consumed tokens: 2243952640 | elapsed time per iteration (s): 5.40 | learning rate: 5.280E-04 | global batch size: 512 | lm loss: 3.252722E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.824 | TFLOPs: 71.62 | +[default7]: iteration 2141/ 3814 | consumed samples: 1096192 | consumed tokens: 2245001216 | elapsed time per iteration (s): 5.43 | learning rate: 5.276E-04 | global batch size: 512 | lm loss: 3.257790E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.376 | TFLOPs: 71.28 | +[default7]: iteration 2142/ 3814 | consumed samples: 1096704 | consumed tokens: 2246049792 | elapsed time per iteration (s): 5.42 | learning rate: 5.272E-04 | global batch size: 512 | lm loss: 3.270201E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.441 | TFLOPs: 71.33 | +[default7]: iteration 2143/ 3814 | consumed samples: 1097216 | consumed tokens: 2247098368 | elapsed time per iteration (s): 5.41 | learning rate: 5.268E-04 | global batch size: 512 | lm loss: 3.268770E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.673 | TFLOPs: 71.51 | +[default7]: iteration 2144/ 3814 | consumed samples: 1097728 | consumed tokens: 2248146944 | elapsed time per iteration (s): 5.39 | learning rate: 5.264E-04 | global batch size: 512 | lm loss: 3.265879E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.956 | TFLOPs: 71.72 | +[default7]: iteration 2145/ 3814 | consumed samples: 1098240 | consumed tokens: 2249195520 | elapsed time per iteration (s): 5.40 | learning rate: 5.260E-04 | global batch size: 512 | lm loss: 3.266789E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.902 | TFLOPs: 71.68 | +[default7]: iteration 2146/ 3814 | consumed samples: 1098752 | consumed tokens: 2250244096 | elapsed time per iteration (s): 5.39 | learning rate: 5.255E-04 | global batch size: 512 | lm loss: 3.277821E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.052 | TFLOPs: 71.79 | +[default7]: iteration 2147/ 3814 | consumed samples: 1099264 | consumed tokens: 2251292672 | elapsed time per iteration (s): 5.40 | learning rate: 5.251E-04 | global batch size: 512 | lm loss: 3.267141E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.886 | TFLOPs: 71.67 | +[default7]: iteration 2148/ 3814 | consumed samples: 1099776 | consumed tokens: 2252341248 | elapsed time per iteration (s): 5.41 | learning rate: 5.247E-04 | global batch size: 512 | lm loss: 3.277515E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.667 | TFLOPs: 71.50 | +[default7]: iteration 2149/ 3814 | consumed samples: 1100288 | consumed tokens: 2253389824 | elapsed time per iteration (s): 5.40 | learning rate: 5.243E-04 | global batch size: 512 | lm loss: 3.239138E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.897 | TFLOPs: 71.68 | +[default7]: iteration 2150/ 3814 | consumed samples: 1100800 | consumed tokens: 2254438400 | elapsed time per iteration (s): 5.43 | learning rate: 5.239E-04 | global batch size: 512 | lm loss: 3.279572E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.317 | TFLOPs: 71.24 | +[default7]: iteration 2151/ 3814 | consumed samples: 1101312 | consumed tokens: 2255486976 | elapsed time per iteration (s): 5.41 | learning rate: 5.235E-04 | global batch size: 512 | lm loss: 3.269156E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.716 | TFLOPs: 71.54 | +[default7]: iteration 2152/ 3814 | consumed samples: 1101824 | consumed tokens: 2256535552 | elapsed time per iteration (s): 5.41 | learning rate: 5.231E-04 | global batch size: 512 | lm loss: 3.260617E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.672 | TFLOPs: 71.51 | +[default7]: iteration 2153/ 3814 | consumed samples: 1102336 | consumed tokens: 2257584128 | elapsed time per iteration (s): 5.43 | learning rate: 5.227E-04 | global batch size: 512 | lm loss: 3.255684E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 2154/ 3814 | consumed samples: 1102848 | consumed tokens: 2258632704 | elapsed time per iteration (s): 5.41 | learning rate: 5.223E-04 | global batch size: 512 | lm loss: 3.264593E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 2155/ 3814 | consumed samples: 1103360 | consumed tokens: 2259681280 | elapsed time per iteration (s): 5.41 | learning rate: 5.219E-04 | global batch size: 512 | lm loss: 3.260658E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.575 | TFLOPs: 71.43 | +[default7]: iteration 2156/ 3814 | consumed samples: 1103872 | consumed tokens: 2260729856 | elapsed time per iteration (s): 5.40 | learning rate: 5.215E-04 | global batch size: 512 | lm loss: 3.297067E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 2157/ 3814 | consumed samples: 1104384 | consumed tokens: 2261778432 | elapsed time per iteration (s): 5.41 | learning rate: 5.211E-04 | global batch size: 512 | lm loss: 3.270491E+00 | loss scale: 65536.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 2158/ 3814 | consumed samples: 1104896 | consumed tokens: 2262827008 | elapsed time per iteration (s): 5.42 | learning rate: 5.206E-04 | global batch size: 512 | lm loss: 3.270283E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 2159/ 3814 | consumed samples: 1105408 | consumed tokens: 2263875584 | elapsed time per iteration (s): 5.42 | learning rate: 5.202E-04 | global batch size: 512 | lm loss: 3.258113E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.483 | TFLOPs: 71.36 | +[default7]: iteration 2160/ 3814 | consumed samples: 1105920 | consumed tokens: 2264924160 | elapsed time per iteration (s): 5.40 | learning rate: 5.198E-04 | global batch size: 512 | lm loss: 3.269079E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.757 | TFLOPs: 71.57 | +[default7]: iteration 2161/ 3814 | consumed samples: 1106432 | consumed tokens: 2265972736 | elapsed time per iteration (s): 5.41 | learning rate: 5.194E-04 | global batch size: 512 | lm loss: 3.269565E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 2162/ 3814 | consumed samples: 1106944 | consumed tokens: 2267021312 | elapsed time per iteration (s): 5.42 | learning rate: 5.190E-04 | global batch size: 512 | lm loss: 3.255401E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.484 | TFLOPs: 71.36 | +[default7]: iteration 2163/ 3814 | consumed samples: 1107456 | consumed tokens: 2268069888 | elapsed time per iteration (s): 5.41 | learning rate: 5.186E-04 | global batch size: 512 | lm loss: 3.281581E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.605 | TFLOPs: 71.46 | +[default7]: iteration 2164/ 3814 | consumed samples: 1107968 | consumed tokens: 2269118464 | elapsed time per iteration (s): 5.41 | learning rate: 5.182E-04 | global batch size: 512 | lm loss: 3.257138E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.574 | TFLOPs: 71.43 | +[default7]: iteration 2165/ 3814 | consumed samples: 1108480 | consumed tokens: 2270167040 | elapsed time per iteration (s): 5.40 | learning rate: 5.178E-04 | global batch size: 512 | lm loss: 3.279813E+00 | loss scale: 65536.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.802 | TFLOPs: 71.60 | +[default7]: iteration 2166/ 3814 | consumed samples: 1108992 | consumed tokens: 2271215616 | elapsed time per iteration (s): 5.40 | learning rate: 5.174E-04 | global batch size: 512 | lm loss: 3.253587E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.877 | TFLOPs: 71.66 | +[default7]: iteration 2167/ 3814 | consumed samples: 1109504 | consumed tokens: 2272264192 | elapsed time per iteration (s): 5.45 | learning rate: 5.170E-04 | global batch size: 512 | lm loss: 3.258739E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.914 | TFLOPs: 70.93 | +[default7]: iteration 2168/ 3814 | consumed samples: 1110016 | consumed tokens: 2273312768 | elapsed time per iteration (s): 5.42 | learning rate: 5.166E-04 | global batch size: 512 | lm loss: 3.234093E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 2169/ 3814 | consumed samples: 1110528 | consumed tokens: 2274361344 | elapsed time per iteration (s): 5.40 | learning rate: 5.162E-04 | global batch size: 512 | lm loss: 3.259677E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.813 | TFLOPs: 71.61 | +[default7]: iteration 2170/ 3814 | consumed samples: 1111040 | consumed tokens: 2275409920 | elapsed time per iteration (s): 5.40 | learning rate: 5.157E-04 | global batch size: 512 | lm loss: 3.236192E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.737 | TFLOPs: 71.56 | +[default7]: iteration 2171/ 3814 | consumed samples: 1111552 | consumed tokens: 2276458496 | elapsed time per iteration (s): 5.42 | learning rate: 5.153E-04 | global batch size: 512 | lm loss: 3.254923E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.472 | TFLOPs: 71.36 | +[default7]: iteration 2172/ 3814 | consumed samples: 1112064 | consumed tokens: 2277507072 | elapsed time per iteration (s): 5.41 | learning rate: 5.149E-04 | global batch size: 512 | lm loss: 3.232287E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.660 | TFLOPs: 71.50 | +[default7]: iteration 2173/ 3814 | consumed samples: 1112576 | consumed tokens: 2278555648 | elapsed time per iteration (s): 5.40 | learning rate: 5.145E-04 | global batch size: 512 | lm loss: 3.277873E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.739 | TFLOPs: 71.56 | +[default7]: iteration 2174/ 3814 | consumed samples: 1113088 | consumed tokens: 2279604224 | elapsed time per iteration (s): 5.40 | learning rate: 5.141E-04 | global batch size: 512 | lm loss: 3.278033E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.883 | TFLOPs: 71.67 | +[default7]: iteration 2175/ 3814 | consumed samples: 1113600 | consumed tokens: 2280652800 | elapsed time per iteration (s): 5.41 | learning rate: 5.137E-04 | global batch size: 512 | lm loss: 3.261350E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.605 | TFLOPs: 71.46 | +[default7]: iteration 2176/ 3814 | consumed samples: 1114112 | consumed tokens: 2281701376 | elapsed time per iteration (s): 5.40 | learning rate: 5.133E-04 | global batch size: 512 | lm loss: 3.238960E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.749 | TFLOPs: 71.56 | +[default7]: iteration 2177/ 3814 | consumed samples: 1114624 | consumed tokens: 2282749952 | elapsed time per iteration (s): 5.41 | learning rate: 5.129E-04 | global batch size: 512 | lm loss: 3.276441E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.557 | TFLOPs: 71.42 | +[default7]: iteration 2178/ 3814 | consumed samples: 1115136 | consumed tokens: 2283798528 | elapsed time per iteration (s): 5.40 | learning rate: 5.125E-04 | global batch size: 512 | lm loss: 3.297092E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.810 | TFLOPs: 71.61 | +[default7]: iteration 2179/ 3814 | consumed samples: 1115648 | consumed tokens: 2284847104 | elapsed time per iteration (s): 5.39 | learning rate: 5.121E-04 | global batch size: 512 | lm loss: 3.269834E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.930 | TFLOPs: 71.70 | +[default7]: iteration 2180/ 3814 | consumed samples: 1116160 | consumed tokens: 2285895680 | elapsed time per iteration (s): 5.39 | learning rate: 5.117E-04 | global batch size: 512 | lm loss: 3.256196E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.925 | TFLOPs: 71.70 | +[default7]: iteration 2181/ 3814 | consumed samples: 1116672 | consumed tokens: 2286944256 | elapsed time per iteration (s): 5.41 | learning rate: 5.113E-04 | global batch size: 512 | lm loss: 3.269158E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.610 | TFLOPs: 71.46 | +[default7]: iteration 2182/ 3814 | consumed samples: 1117184 | consumed tokens: 2287992832 | elapsed time per iteration (s): 5.40 | learning rate: 5.109E-04 | global batch size: 512 | lm loss: 3.247858E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.780 | TFLOPs: 71.59 | +[default7]: iteration 2183/ 3814 | consumed samples: 1117696 | consumed tokens: 2289041408 | elapsed time per iteration (s): 5.40 | learning rate: 5.105E-04 | global batch size: 512 | lm loss: 3.240094E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.778 | TFLOPs: 71.59 | +[default7]: iteration 2184/ 3814 | consumed samples: 1118208 | consumed tokens: 2290089984 | elapsed time per iteration (s): 5.40 | learning rate: 5.100E-04 | global batch size: 512 | lm loss: 3.279031E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.879 | TFLOPs: 71.66 | +[default7]: iteration 2185/ 3814 | consumed samples: 1118720 | consumed tokens: 2291138560 | elapsed time per iteration (s): 5.42 | learning rate: 5.096E-04 | global batch size: 512 | lm loss: 3.258363E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.519 | TFLOPs: 71.39 | +[default7]: iteration 2186/ 3814 | consumed samples: 1119232 | consumed tokens: 2292187136 | elapsed time per iteration (s): 5.40 | learning rate: 5.092E-04 | global batch size: 512 | lm loss: 3.278671E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.856 | TFLOPs: 71.65 | +[default7]: iteration 2187/ 3814 | consumed samples: 1119744 | consumed tokens: 2293235712 | elapsed time per iteration (s): 5.41 | learning rate: 5.088E-04 | global batch size: 512 | lm loss: 3.283306E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.697 | TFLOPs: 71.53 | +[default7]: iteration 2188/ 3814 | consumed samples: 1120256 | consumed tokens: 2294284288 | elapsed time per iteration (s): 5.43 | learning rate: 5.084E-04 | global batch size: 512 | lm loss: 3.259862E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.342 | TFLOPs: 71.26 | +[default7]: iteration 2189/ 3814 | consumed samples: 1120768 | consumed tokens: 2295332864 | elapsed time per iteration (s): 5.42 | learning rate: 5.080E-04 | global batch size: 512 | lm loss: 3.278060E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.415 | TFLOPs: 71.31 | +[default7]: iteration 2190/ 3814 | consumed samples: 1121280 | consumed tokens: 2296381440 | elapsed time per iteration (s): 5.43 | learning rate: 5.076E-04 | global batch size: 512 | lm loss: 3.261118E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 2191/ 3814 | consumed samples: 1121792 | consumed tokens: 2297430016 | elapsed time per iteration (s): 5.42 | learning rate: 5.072E-04 | global batch size: 512 | lm loss: 3.260475E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.509 | TFLOPs: 71.38 | +[default7]: iteration 2192/ 3814 | consumed samples: 1122304 | consumed tokens: 2298478592 | elapsed time per iteration (s): 5.40 | learning rate: 5.068E-04 | global batch size: 512 | lm loss: 3.261261E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.746 | TFLOPs: 71.56 | +[default7]: iteration 2193/ 3814 | consumed samples: 1122816 | consumed tokens: 2299527168 | elapsed time per iteration (s): 5.41 | learning rate: 5.064E-04 | global batch size: 512 | lm loss: 3.285741E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.602 | TFLOPs: 71.45 | +[default7]: iteration 2194/ 3814 | consumed samples: 1123328 | consumed tokens: 2300575744 | elapsed time per iteration (s): 5.41 | learning rate: 5.060E-04 | global batch size: 512 | lm loss: 3.271036E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.561 | TFLOPs: 71.42 | +[default7]: iteration 2195/ 3814 | consumed samples: 1123840 | consumed tokens: 2301624320 | elapsed time per iteration (s): 5.43 | learning rate: 5.056E-04 | global batch size: 512 | lm loss: 3.280423E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.270 | TFLOPs: 71.20 | +[default7]: iteration 2196/ 3814 | consumed samples: 1124352 | consumed tokens: 2302672896 | elapsed time per iteration (s): 5.44 | learning rate: 5.052E-04 | global batch size: 512 | lm loss: 3.239801E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.169 | TFLOPs: 71.13 | +[default7]: iteration 2197/ 3814 | consumed samples: 1124864 | consumed tokens: 2303721472 | elapsed time per iteration (s): 5.42 | learning rate: 5.048E-04 | global batch size: 512 | lm loss: 3.260433E+00 | loss scale: 65536.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.429 | TFLOPs: 71.32 | +[default7]: iteration 2198/ 3814 | consumed samples: 1125376 | consumed tokens: 2304770048 | elapsed time per iteration (s): 5.42 | learning rate: 5.043E-04 | global batch size: 512 | lm loss: 3.239275E+00 | loss scale: 65536.0 | grad norm: 0.089 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.462 | TFLOPs: 71.35 | +[default7]: iteration 2199/ 3814 | consumed samples: 1125888 | consumed tokens: 2305818624 | elapsed time per iteration (s): 5.43 | learning rate: 5.039E-04 | global batch size: 512 | lm loss: 3.212590E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.300 | TFLOPs: 71.23 | +[default7]: iteration 2200/ 3814 | consumed samples: 1126400 | consumed tokens: 2306867200 | elapsed time per iteration (s): 5.43 | learning rate: 5.035E-04 | global batch size: 512 | lm loss: 3.260522E+00 | loss scale: 65536.0 | grad norm: 0.088 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.284 | TFLOPs: 71.21 | +[default7]: iteration 2201/ 3814 | consumed samples: 1126912 | consumed tokens: 2307915776 | elapsed time per iteration (s): 5.42 | learning rate: 5.031E-04 | global batch size: 512 | lm loss: 3.246857E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.475 | TFLOPs: 71.36 | +[default7]: iteration 2202/ 3814 | consumed samples: 1127424 | consumed tokens: 2308964352 | elapsed time per iteration (s): 5.43 | learning rate: 5.027E-04 | global batch size: 512 | lm loss: 3.261820E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.217 | TFLOPs: 71.16 | +[default7]: iteration 2203/ 3814 | consumed samples: 1127936 | consumed tokens: 2310012928 | elapsed time per iteration (s): 5.42 | learning rate: 5.023E-04 | global batch size: 512 | lm loss: 3.242143E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.402 | TFLOPs: 71.30 | +[default7]: iteration 2204/ 3814 | consumed samples: 1128448 | consumed tokens: 2311061504 | elapsed time per iteration (s): 5.43 | learning rate: 5.019E-04 | global batch size: 512 | lm loss: 3.231701E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.217 | TFLOPs: 71.16 | +[default7]: iteration 2205/ 3814 | consumed samples: 1128960 | consumed tokens: 2312110080 | elapsed time per iteration (s): 5.42 | learning rate: 5.015E-04 | global batch size: 512 | lm loss: 3.265965E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.477 | TFLOPs: 71.36 | +[default7]: iteration 2206/ 3814 | consumed samples: 1129472 | consumed tokens: 2313158656 | elapsed time per iteration (s): 5.44 | learning rate: 5.011E-04 | global batch size: 512 | lm loss: 3.276862E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.158 | TFLOPs: 71.12 | +[default7]: iteration 2207/ 3814 | consumed samples: 1129984 | consumed tokens: 2314207232 | elapsed time per iteration (s): 5.42 | learning rate: 5.007E-04 | global batch size: 512 | lm loss: 3.242825E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.418 | TFLOPs: 71.31 | +[default7]: iteration 2208/ 3814 | consumed samples: 1130496 | consumed tokens: 2315255808 | elapsed time per iteration (s): 5.44 | learning rate: 5.003E-04 | global batch size: 512 | lm loss: 3.251964E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.173 | TFLOPs: 71.13 | +[default7]: iteration 2209/ 3814 | consumed samples: 1131008 | consumed tokens: 2316304384 | elapsed time per iteration (s): 5.44 | learning rate: 4.999E-04 | global batch size: 512 | lm loss: 3.255703E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.119 | TFLOPs: 71.09 | +[default7]: iteration 2210/ 3814 | consumed samples: 1131520 | consumed tokens: 2317352960 | elapsed time per iteration (s): 5.42 | learning rate: 4.995E-04 | global batch size: 512 | lm loss: 3.254624E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.396 | TFLOPs: 71.30 | +[default7]: iteration 2211/ 3814 | consumed samples: 1132032 | consumed tokens: 2318401536 | elapsed time per iteration (s): 5.46 | learning rate: 4.991E-04 | global batch size: 512 | lm loss: 3.231002E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.796 | TFLOPs: 70.84 | +[default7]: iteration 2212/ 3814 | consumed samples: 1132544 | consumed tokens: 2319450112 | elapsed time per iteration (s): 5.43 | learning rate: 4.987E-04 | global batch size: 512 | lm loss: 3.261494E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.241 | TFLOPs: 71.18 | +[default7]: iteration 2213/ 3814 | consumed samples: 1133056 | consumed tokens: 2320498688 | elapsed time per iteration (s): 5.41 | learning rate: 4.982E-04 | global batch size: 512 | lm loss: 3.267773E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.564 | TFLOPs: 71.43 | +[default7]: iteration 2214/ 3814 | consumed samples: 1133568 | consumed tokens: 2321547264 | elapsed time per iteration (s): 5.42 | learning rate: 4.978E-04 | global batch size: 512 | lm loss: 3.248943E+00 | loss scale: 65536.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.425 | TFLOPs: 71.32 | +[default7]: iteration 2215/ 3814 | consumed samples: 1134080 | consumed tokens: 2322595840 | elapsed time per iteration (s): 5.40 | learning rate: 4.974E-04 | global batch size: 512 | lm loss: 3.258320E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.885 | TFLOPs: 71.67 | +[default7]: iteration 2216/ 3814 | consumed samples: 1134592 | consumed tokens: 2323644416 | elapsed time per iteration (s): 5.40 | learning rate: 4.970E-04 | global batch size: 512 | lm loss: 3.233402E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.810 | TFLOPs: 71.61 | +[default7]: iteration 2217/ 3814 | consumed samples: 1135104 | consumed tokens: 2324692992 | elapsed time per iteration (s): 5.40 | learning rate: 4.966E-04 | global batch size: 512 | lm loss: 3.235782E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.809 | TFLOPs: 71.61 | +[default7]: iteration 2218/ 3814 | consumed samples: 1135616 | consumed tokens: 2325741568 | elapsed time per iteration (s): 5.41 | learning rate: 4.962E-04 | global batch size: 512 | lm loss: 3.265632E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.577 | TFLOPs: 71.44 | +[default7]: iteration 2219/ 3814 | consumed samples: 1136128 | consumed tokens: 2326790144 | elapsed time per iteration (s): 5.41 | learning rate: 4.958E-04 | global batch size: 512 | lm loss: 3.226298E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.714 | TFLOPs: 71.54 | +[default7]: iteration 2220/ 3814 | consumed samples: 1136640 | consumed tokens: 2327838720 | elapsed time per iteration (s): 5.42 | learning rate: 4.954E-04 | global batch size: 512 | lm loss: 3.233435E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.550 | TFLOPs: 71.41 | +[default7]: iteration 2221/ 3814 | consumed samples: 1137152 | consumed tokens: 2328887296 | elapsed time per iteration (s): 5.41 | learning rate: 4.950E-04 | global batch size: 512 | lm loss: 3.288350E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.575 | TFLOPs: 71.43 | +[default7]: iteration 2222/ 3814 | consumed samples: 1137664 | consumed tokens: 2329935872 | elapsed time per iteration (s): 5.42 | learning rate: 4.946E-04 | global batch size: 512 | lm loss: 3.232152E+00 | loss scale: 65536.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default7]: iteration 2223/ 3814 | consumed samples: 1138176 | consumed tokens: 2330984448 | elapsed time per iteration (s): 5.41 | learning rate: 4.942E-04 | global batch size: 512 | lm loss: 3.235996E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.614 | TFLOPs: 71.46 | +[default7]: iteration 2224/ 3814 | consumed samples: 1138688 | consumed tokens: 2332033024 | elapsed time per iteration (s): 5.42 | learning rate: 4.938E-04 | global batch size: 512 | lm loss: 3.271413E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.41 | +[default7]: iteration 2225/ 3814 | consumed samples: 1139200 | consumed tokens: 2333081600 | elapsed time per iteration (s): 5.41 | learning rate: 4.934E-04 | global batch size: 512 | lm loss: 3.281901E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.649 | TFLOPs: 71.49 | +[default7]: iteration 2226/ 3814 | consumed samples: 1139712 | consumed tokens: 2334130176 | elapsed time per iteration (s): 5.41 | learning rate: 4.930E-04 | global batch size: 512 | lm loss: 3.231486E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.580 | TFLOPs: 71.44 | +[default7]: iteration 2227/ 3814 | consumed samples: 1140224 | consumed tokens: 2335178752 | elapsed time per iteration (s): 5.43 | learning rate: 4.926E-04 | global batch size: 512 | lm loss: 3.244244E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.367 | TFLOPs: 71.28 | +[default7]: iteration 2228/ 3814 | consumed samples: 1140736 | consumed tokens: 2336227328 | elapsed time per iteration (s): 5.41 | learning rate: 4.922E-04 | global batch size: 512 | lm loss: 3.265047E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.593 | TFLOPs: 71.45 | +[default7]: iteration 2229/ 3814 | consumed samples: 1141248 | consumed tokens: 2337275904 | elapsed time per iteration (s): 5.41 | learning rate: 4.918E-04 | global batch size: 512 | lm loss: 3.237251E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 2230/ 3814 | consumed samples: 1141760 | consumed tokens: 2338324480 | elapsed time per iteration (s): 5.41 | learning rate: 4.913E-04 | global batch size: 512 | lm loss: 3.244096E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.586 | TFLOPs: 71.44 | +[default7]: iteration 2231/ 3814 | consumed samples: 1142272 | consumed tokens: 2339373056 | elapsed time per iteration (s): 5.42 | learning rate: 4.909E-04 | global batch size: 512 | lm loss: 3.207394E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.502 | TFLOPs: 71.38 | +[default7]: iteration 2232/ 3814 | consumed samples: 1142784 | consumed tokens: 2340421632 | elapsed time per iteration (s): 5.42 | learning rate: 4.905E-04 | global batch size: 512 | lm loss: 3.241956E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.462 | TFLOPs: 71.35 | +[default7]: iteration 2233/ 3814 | consumed samples: 1143296 | consumed tokens: 2341470208 | elapsed time per iteration (s): 5.42 | learning rate: 4.901E-04 | global batch size: 512 | lm loss: 3.246994E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 2234/ 3814 | consumed samples: 1143808 | consumed tokens: 2342518784 | elapsed time per iteration (s): 5.43 | learning rate: 4.897E-04 | global batch size: 512 | lm loss: 3.257237E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 2235/ 3814 | consumed samples: 1144320 | consumed tokens: 2343567360 | elapsed time per iteration (s): 5.41 | learning rate: 4.893E-04 | global batch size: 512 | lm loss: 3.243950E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.626 | TFLOPs: 71.47 | +[default7]: iteration 2236/ 3814 | consumed samples: 1144832 | consumed tokens: 2344615936 | elapsed time per iteration (s): 5.42 | learning rate: 4.889E-04 | global batch size: 512 | lm loss: 3.275307E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.527 | TFLOPs: 71.40 | +[default7]: iteration 2237/ 3814 | consumed samples: 1145344 | consumed tokens: 2345664512 | elapsed time per iteration (s): 5.42 | learning rate: 4.885E-04 | global batch size: 512 | lm loss: 3.227118E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.519 | TFLOPs: 71.39 | +[default7]: iteration 2238/ 3814 | consumed samples: 1145856 | consumed tokens: 2346713088 | elapsed time per iteration (s): 5.42 | learning rate: 4.881E-04 | global batch size: 512 | lm loss: 3.256396E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.519 | TFLOPs: 71.39 | +[default7]: iteration 2239/ 3814 | consumed samples: 1146368 | consumed tokens: 2347761664 | elapsed time per iteration (s): 5.42 | learning rate: 4.877E-04 | global batch size: 512 | lm loss: 3.256049E+00 | loss scale: 65536.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.411 | TFLOPs: 71.31 | +[default7]: iteration 2240/ 3814 | consumed samples: 1146880 | consumed tokens: 2348810240 | elapsed time per iteration (s): 5.42 | learning rate: 4.873E-04 | global batch size: 512 | lm loss: 3.229876E+00 | loss scale: 65536.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.494 | TFLOPs: 71.37 | +[default7]: iteration 2241/ 3814 | consumed samples: 1147392 | consumed tokens: 2349858816 | elapsed time per iteration (s): 5.43 | learning rate: 4.869E-04 | global batch size: 512 | lm loss: 3.245943E+00 | loss scale: 65536.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.272 | TFLOPs: 71.20 | +[default7]: iteration 2242/ 3814 | consumed samples: 1147904 | consumed tokens: 2350907392 | elapsed time per iteration (s): 5.42 | learning rate: 4.865E-04 | global batch size: 512 | lm loss: 3.239116E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.520 | TFLOPs: 71.39 | +[default7]: iteration 2243/ 3814 | consumed samples: 1148416 | consumed tokens: 2351955968 | elapsed time per iteration (s): 5.42 | learning rate: 4.861E-04 | global batch size: 512 | lm loss: 3.230335E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.465 | TFLOPs: 71.35 | +[default7]: iteration 2244/ 3814 | consumed samples: 1148928 | consumed tokens: 2353004544 | elapsed time per iteration (s): 5.41 | learning rate: 4.857E-04 | global batch size: 512 | lm loss: 3.262447E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.559 | TFLOPs: 71.42 | +[default7]: iteration 2245/ 3814 | consumed samples: 1149440 | consumed tokens: 2354053120 | elapsed time per iteration (s): 5.41 | learning rate: 4.853E-04 | global batch size: 512 | lm loss: 3.244488E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.725 | TFLOPs: 71.55 | +[default7]: iteration 2246/ 3814 | consumed samples: 1149952 | consumed tokens: 2355101696 | elapsed time per iteration (s): 5.41 | learning rate: 4.849E-04 | global batch size: 512 | lm loss: 3.246363E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.641 | TFLOPs: 71.48 | +[default7]: iteration 2247/ 3814 | consumed samples: 1150464 | consumed tokens: 2356150272 | elapsed time per iteration (s): 5.40 | learning rate: 4.845E-04 | global batch size: 512 | lm loss: 3.248917E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.792 | TFLOPs: 71.60 | +[default7]: iteration 2248/ 3814 | consumed samples: 1150976 | consumed tokens: 2357198848 | elapsed time per iteration (s): 5.41 | learning rate: 4.841E-04 | global batch size: 512 | lm loss: 3.246160E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.686 | TFLOPs: 71.52 | +[default7]: iteration 2249/ 3814 | consumed samples: 1151488 | consumed tokens: 2358247424 | elapsed time per iteration (s): 5.41 | learning rate: 4.837E-04 | global batch size: 512 | lm loss: 3.197200E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.661 | TFLOPs: 71.50 | +[default1]:[2023-02-16 09:43:08,140] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_01-model_01-model_states.pt... +[default7]: iteration 2250/ 3814 | consumed samples: 1152000 | consumed tokens: 2359296000 | elapsed time per iteration (s): 5.41 | learning rate: 4.832E-04 | global batch size: 512 | lm loss: 3.218948E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.622 | TFLOPs: 71.47 | +[default0]:saving checkpoint at iteration 2250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 09:43:08,135] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2250 is begin to save! +[default0]:[2023-02-16 09:43:08,140] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_01-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:08,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,288] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,289] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_04-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:08,359] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,359] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,358] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,359] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_06-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,435] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,436] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:08,503] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,503] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,493] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,494] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_07-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:08,550] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,550] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_08-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,559] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:08,615] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,615] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,606] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,607] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,680] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,681] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,666] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,667] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:08,734] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,735] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,743] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_11-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:08,797] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,797] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,791] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,792] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:08,845] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,846] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,857] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:08,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,921] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,908] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:08,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:08,978] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:08,965] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:08,966] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,022] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,023] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,037] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_16-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,100] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,086] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,086] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,147] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,147] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,169] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,169] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,206] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,206] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_19-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,229] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,229] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,262] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,263] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_20-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,321] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_21-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,344] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_21-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,398] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,398] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,377] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_22-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,435] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,458] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,459] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_23-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,513] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,514] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,491] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,492] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,546] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,546] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,567] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,568] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,601] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,601] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_26-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,656] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,656] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,624] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,624] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_26-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,683] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,684] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,719] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,719] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,741] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,742] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_28-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,802] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,787] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,788] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,851] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,864] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,865] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:09,919] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,920] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:09,907] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,907] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:09,958] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:09,959] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:09,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:09,982] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:10,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:10,019] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:10,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:10,038] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_33-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:10,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:10,091] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 09:43:10,073] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:10,074] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:10,129] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:10,129] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_35-model_01-model_states.pt... +[default0]:[2023-02-16 09:43:10,155] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:10,155] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 09:43:10,209] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:10,210] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_37-model_00-model_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:43:10,211] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 09:43:10,212] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_00_model_states.pt +[default0]:[2023-02-16 09:43:10,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 09:43:10,214] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_00_model_states.pt. +[default1]:[2023-02-16 09:43:10,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:10,194] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 09:43:10,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 09:43:10,196] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_01_model_states.pt +[default1]:[2023-02-16 09:43:10,196] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 09:43:10,198] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 09:43:10,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 09:43:10,752] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,753] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,753] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,740] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,740] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,740] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,725] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,725] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,738] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,739] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,739] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,714] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,714] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,714] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,769] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,769] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,769] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,777] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,777] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,777] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,733] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,733] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,743] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,744] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,771] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,771] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,731] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,731] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,731] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,753] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,754] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,755] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,783] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,783] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,756] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,756] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,829] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,829] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,829] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,820] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,820] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,820] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,821] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,821] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,830] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,830] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,830] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,846] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,846] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,846] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,819] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,819] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,819] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,853] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,853] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,853] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,838] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,838] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default4]:[2023-02-16 09:43:10,772] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,772] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,772] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,841] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,841] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,850] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,850] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,816] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,816] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,858] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,858] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,821] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,821] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,850] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,850] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,850] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,816] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,816] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,882] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,899] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,899] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,842] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,842] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,823] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,823] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,823] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,903] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,903] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,848] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,849] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,849] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,889] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,895] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,895] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,895] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,896] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,897] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,897] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,918] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,918] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,859] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,859] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,856] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,856] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,894] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,894] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,897] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,898] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,898] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,913] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,913] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,923] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,923] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,868] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,868] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,878] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,878] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,904] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,904] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,904] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,882] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,948] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,948] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,967] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,967] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,929] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,929] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,929] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default1]:[2023-02-16 09:43:10,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 09:43:10,921] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 09:43:10,921] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,944] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,944] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,944] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default2]:[2023-02-16 09:43:10,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 09:43:10,947] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 09:43:10,947] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:[2023-02-16 09:43:10,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 09:43:10,923] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 09:43:10,923] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,986] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,986] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,986] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]:[2023-02-16 09:43:10,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 09:43:10,930] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 09:43:10,930] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default0]: successfully saved checkpoint at iteration 2250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default4]:[2023-02-16 09:43:10,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 09:43:10,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 09:43:10,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default6]:[2023-02-16 09:43:10,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 09:43:10,932] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 09:43:10,932] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default3]:[2023-02-16 09:43:10,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 09:43:10,963] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 09:43:10,963] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default5]:[2023-02-16 09:43:10,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 09:43:10,994] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2250/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 09:43:10,994] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2250 is ready now! +[default7]:time (ms) | save-checkpoint: 2860.52 +[default7]: iteration 2251/ 3814 | consumed samples: 1152512 | consumed tokens: 2360344576 | elapsed time per iteration (s): 8.27 | learning rate: 4.828E-04 | global batch size: 512 | lm loss: 3.240197E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.913 | TFLOPs: 46.76 | +[default7]: iteration 2252/ 3814 | consumed samples: 1153024 | consumed tokens: 2361393152 | elapsed time per iteration (s): 5.41 | learning rate: 4.824E-04 | global batch size: 512 | lm loss: 3.225081E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.667 | TFLOPs: 71.50 | +[default7]: iteration 2253/ 3814 | consumed samples: 1153536 | consumed tokens: 2362441728 | elapsed time per iteration (s): 5.41 | learning rate: 4.820E-04 | global batch size: 512 | lm loss: 3.232363E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 2254/ 3814 | consumed samples: 1154048 | consumed tokens: 2363490304 | elapsed time per iteration (s): 5.43 | learning rate: 4.816E-04 | global batch size: 512 | lm loss: 3.242476E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.369 | TFLOPs: 71.28 | +[default7]: iteration 2255/ 3814 | consumed samples: 1154560 | consumed tokens: 2364538880 | elapsed time per iteration (s): 5.42 | learning rate: 4.812E-04 | global batch size: 512 | lm loss: 3.241773E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.441 | TFLOPs: 71.33 | +[default7]: iteration 2256/ 3814 | consumed samples: 1155072 | consumed tokens: 2365587456 | elapsed time per iteration (s): 5.41 | learning rate: 4.808E-04 | global batch size: 512 | lm loss: 3.229866E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.671 | TFLOPs: 71.51 | +[default7]: iteration 2257/ 3814 | consumed samples: 1155584 | consumed tokens: 2366636032 | elapsed time per iteration (s): 5.42 | learning rate: 4.804E-04 | global batch size: 512 | lm loss: 3.234726E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.406 | TFLOPs: 71.31 | +[default7]: iteration 2258/ 3814 | consumed samples: 1156096 | consumed tokens: 2367684608 | elapsed time per iteration (s): 5.41 | learning rate: 4.800E-04 | global batch size: 512 | lm loss: 3.230846E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.664 | TFLOPs: 71.50 | +[default7]: iteration 2259/ 3814 | consumed samples: 1156608 | consumed tokens: 2368733184 | elapsed time per iteration (s): 5.41 | learning rate: 4.796E-04 | global batch size: 512 | lm loss: 3.248386E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.626 | TFLOPs: 71.47 | +[default7]: iteration 2260/ 3814 | consumed samples: 1157120 | consumed tokens: 2369781760 | elapsed time per iteration (s): 5.40 | learning rate: 4.792E-04 | global batch size: 512 | lm loss: 3.252628E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 2261/ 3814 | consumed samples: 1157632 | consumed tokens: 2370830336 | elapsed time per iteration (s): 5.39 | learning rate: 4.788E-04 | global batch size: 512 | lm loss: 3.218733E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.943 | TFLOPs: 71.71 | +[default7]: iteration 2262/ 3814 | consumed samples: 1158144 | consumed tokens: 2371878912 | elapsed time per iteration (s): 5.41 | learning rate: 4.784E-04 | global batch size: 512 | lm loss: 3.222646E+00 | loss scale: 65536.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.701 | TFLOPs: 71.53 | +[default7]: iteration 2263/ 3814 | consumed samples: 1158656 | consumed tokens: 2372927488 | elapsed time per iteration (s): 5.40 | learning rate: 4.780E-04 | global batch size: 512 | lm loss: 3.226898E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.840 | TFLOPs: 71.63 | +[default7]: iteration 2264/ 3814 | consumed samples: 1159168 | consumed tokens: 2373976064 | elapsed time per iteration (s): 5.40 | learning rate: 4.776E-04 | global batch size: 512 | lm loss: 3.216537E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.882 | TFLOPs: 71.67 | +[default7]: iteration 2265/ 3814 | consumed samples: 1159680 | consumed tokens: 2375024640 | elapsed time per iteration (s): 5.40 | learning rate: 4.772E-04 | global batch size: 512 | lm loss: 3.213800E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.816 | TFLOPs: 71.62 | +[default7]: iteration 2266/ 3814 | consumed samples: 1160192 | consumed tokens: 2376073216 | elapsed time per iteration (s): 5.43 | learning rate: 4.768E-04 | global batch size: 512 | lm loss: 3.230367E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.251 | TFLOPs: 71.19 | +[default7]: iteration 2267/ 3814 | consumed samples: 1160704 | consumed tokens: 2377121792 | elapsed time per iteration (s): 5.40 | learning rate: 4.764E-04 | global batch size: 512 | lm loss: 3.283161E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.802 | TFLOPs: 71.60 | +[default7]: iteration 2268/ 3814 | consumed samples: 1161216 | consumed tokens: 2378170368 | elapsed time per iteration (s): 5.40 | learning rate: 4.760E-04 | global batch size: 512 | lm loss: 3.221309E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.778 | TFLOPs: 71.59 | +[default7]: iteration 2269/ 3814 | consumed samples: 1161728 | consumed tokens: 2379218944 | elapsed time per iteration (s): 5.39 | learning rate: 4.756E-04 | global batch size: 512 | lm loss: 3.251762E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.058 | TFLOPs: 71.80 | +[default7]: iteration 2270/ 3814 | consumed samples: 1162240 | consumed tokens: 2380267520 | elapsed time per iteration (s): 5.41 | learning rate: 4.752E-04 | global batch size: 512 | lm loss: 3.227439E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.683 | TFLOPs: 71.51 | +[default7]: iteration 2271/ 3814 | consumed samples: 1162752 | consumed tokens: 2381316096 | elapsed time per iteration (s): 5.39 | learning rate: 4.748E-04 | global batch size: 512 | lm loss: 3.231005E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.903 | TFLOPs: 71.68 | +[default7]: iteration 2272/ 3814 | consumed samples: 1163264 | consumed tokens: 2382364672 | elapsed time per iteration (s): 5.41 | learning rate: 4.744E-04 | global batch size: 512 | lm loss: 3.266022E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 2273/ 3814 | consumed samples: 1163776 | consumed tokens: 2383413248 | elapsed time per iteration (s): 5.40 | learning rate: 4.740E-04 | global batch size: 512 | lm loss: 3.239281E+00 | loss scale: 65536.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.782 | TFLOPs: 71.59 | +[default7]: iteration 2274/ 3814 | consumed samples: 1164288 | consumed tokens: 2384461824 | elapsed time per iteration (s): 5.42 | learning rate: 4.736E-04 | global batch size: 512 | lm loss: 3.253267E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 2275/ 3814 | consumed samples: 1164800 | consumed tokens: 2385510400 | elapsed time per iteration (s): 5.42 | learning rate: 4.732E-04 | global batch size: 512 | lm loss: 3.283349E+00 | loss scale: 65536.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.409 | TFLOPs: 71.31 | +[default7]: iteration 2276/ 3814 | consumed samples: 1165312 | consumed tokens: 2386558976 | elapsed time per iteration (s): 5.42 | learning rate: 4.728E-04 | global batch size: 512 | lm loss: 3.244461E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.412 | TFLOPs: 71.31 | +[default7]: iteration 2277/ 3814 | consumed samples: 1165824 | consumed tokens: 2387607552 | elapsed time per iteration (s): 5.42 | learning rate: 4.724E-04 | global batch size: 512 | lm loss: 3.250588E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.486 | TFLOPs: 71.37 | +[default7]: iteration 2278/ 3814 | consumed samples: 1166336 | consumed tokens: 2388656128 | elapsed time per iteration (s): 5.41 | learning rate: 4.719E-04 | global batch size: 512 | lm loss: 3.238737E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.561 | TFLOPs: 71.42 | +[default7]: iteration 2279/ 3814 | consumed samples: 1166848 | consumed tokens: 2389704704 | elapsed time per iteration (s): 5.41 | learning rate: 4.715E-04 | global batch size: 512 | lm loss: 3.240185E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.644 | TFLOPs: 71.49 | +[default7]: iteration 2280/ 3814 | consumed samples: 1167360 | consumed tokens: 2390753280 | elapsed time per iteration (s): 5.41 | learning rate: 4.711E-04 | global batch size: 512 | lm loss: 3.241297E+00 | loss scale: 65536.0 | grad norm: 0.090 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 2281/ 3814 | consumed samples: 1167872 | consumed tokens: 2391801856 | elapsed time per iteration (s): 5.42 | learning rate: 4.707E-04 | global batch size: 512 | lm loss: 3.250536E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.408 | TFLOPs: 71.31 | +[default7]: iteration 2282/ 3814 | consumed samples: 1168384 | consumed tokens: 2392850432 | elapsed time per iteration (s): 5.41 | learning rate: 4.703E-04 | global batch size: 512 | lm loss: 3.220890E+00 | loss scale: 65536.0 | grad norm: 0.091 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default7]: iteration 2283/ 3814 | consumed samples: 1168896 | consumed tokens: 2393899008 | elapsed time per iteration (s): 5.43 | learning rate: 4.699E-04 | global batch size: 512 | lm loss: 3.213829E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.257 | TFLOPs: 71.19 | +[default7]: iteration 2284/ 3814 | consumed samples: 1169408 | consumed tokens: 2394947584 | elapsed time per iteration (s): 5.42 | learning rate: 4.695E-04 | global batch size: 512 | lm loss: 3.254946E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.498 | TFLOPs: 71.38 | +[default7]: iteration 2285/ 3814 | consumed samples: 1169920 | consumed tokens: 2395996160 | elapsed time per iteration (s): 5.43 | learning rate: 4.691E-04 | global batch size: 512 | lm loss: 3.224355E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.226 | TFLOPs: 71.17 | +[default7]: iteration 2286/ 3814 | consumed samples: 1170432 | consumed tokens: 2397044736 | elapsed time per iteration (s): 5.43 | learning rate: 4.687E-04 | global batch size: 512 | lm loss: 3.222808E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.313 | TFLOPs: 71.24 | +[default7]: iteration 2287/ 3814 | consumed samples: 1170944 | consumed tokens: 2398093312 | elapsed time per iteration (s): 5.42 | learning rate: 4.683E-04 | global batch size: 512 | lm loss: 3.221627E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.387 | TFLOPs: 71.29 | +[default7]: iteration 2288/ 3814 | consumed samples: 1171456 | consumed tokens: 2399141888 | elapsed time per iteration (s): 5.44 | learning rate: 4.679E-04 | global batch size: 512 | lm loss: 3.226715E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.134 | TFLOPs: 71.10 | +[default7]: iteration 2289/ 3814 | consumed samples: 1171968 | consumed tokens: 2400190464 | elapsed time per iteration (s): 5.41 | learning rate: 4.675E-04 | global batch size: 512 | lm loss: 3.235470E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2290/ 3814 | consumed samples: 1172480 | consumed tokens: 2401239040 | elapsed time per iteration (s): 5.41 | learning rate: 4.671E-04 | global batch size: 512 | lm loss: 3.267627E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.653 | TFLOPs: 71.49 | +[default7]: iteration 2291/ 3814 | consumed samples: 1172992 | consumed tokens: 2402287616 | elapsed time per iteration (s): 5.41 | learning rate: 4.667E-04 | global batch size: 512 | lm loss: 3.244025E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.607 | TFLOPs: 71.46 | +[default7]: iteration 2292/ 3814 | consumed samples: 1173504 | consumed tokens: 2403336192 | elapsed time per iteration (s): 5.41 | learning rate: 4.663E-04 | global batch size: 512 | lm loss: 3.242570E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.572 | TFLOPs: 71.43 | +[default7]: iteration 2293/ 3814 | consumed samples: 1174016 | consumed tokens: 2404384768 | elapsed time per iteration (s): 5.41 | learning rate: 4.659E-04 | global batch size: 512 | lm loss: 3.261244E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.649 | TFLOPs: 71.49 | +[default7]: iteration 2294/ 3814 | consumed samples: 1174528 | consumed tokens: 2405433344 | elapsed time per iteration (s): 5.42 | learning rate: 4.655E-04 | global batch size: 512 | lm loss: 3.261322E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.416 | TFLOPs: 71.31 | +[default7]: iteration 2295/ 3814 | consumed samples: 1175040 | consumed tokens: 2406481920 | elapsed time per iteration (s): 5.42 | learning rate: 4.651E-04 | global batch size: 512 | lm loss: 3.266193E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.488 | TFLOPs: 71.37 | +[default7]: iteration 2296/ 3814 | consumed samples: 1175552 | consumed tokens: 2407530496 | elapsed time per iteration (s): 5.44 | learning rate: 4.647E-04 | global batch size: 512 | lm loss: 3.235457E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.174 | TFLOPs: 71.13 | +[default7]: iteration 2297/ 3814 | consumed samples: 1176064 | consumed tokens: 2408579072 | elapsed time per iteration (s): 5.44 | learning rate: 4.643E-04 | global batch size: 512 | lm loss: 3.190844E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.128 | TFLOPs: 71.10 | +[default7]: iteration 2298/ 3814 | consumed samples: 1176576 | consumed tokens: 2409627648 | elapsed time per iteration (s): 5.43 | learning rate: 4.639E-04 | global batch size: 512 | lm loss: 3.245608E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.352 | TFLOPs: 71.26 | +[default7]: iteration 2299/ 3814 | consumed samples: 1177088 | consumed tokens: 2410676224 | elapsed time per iteration (s): 5.43 | learning rate: 4.635E-04 | global batch size: 512 | lm loss: 3.240472E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.282 | TFLOPs: 71.21 | +[default7]: iteration 2300/ 3814 | consumed samples: 1177600 | consumed tokens: 2411724800 | elapsed time per iteration (s): 5.43 | learning rate: 4.631E-04 | global batch size: 512 | lm loss: 3.256798E+00 | loss scale: 65536.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.373 | TFLOPs: 71.28 | +[default7]: iteration 2301/ 3814 | consumed samples: 1178112 | consumed tokens: 2412773376 | elapsed time per iteration (s): 5.43 | learning rate: 4.627E-04 | global batch size: 512 | lm loss: 3.256700E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.311 | TFLOPs: 71.23 | +[default7]: iteration 2302/ 3814 | consumed samples: 1178624 | consumed tokens: 2413821952 | elapsed time per iteration (s): 5.42 | learning rate: 4.623E-04 | global batch size: 512 | lm loss: 3.220416E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.490 | TFLOPs: 71.37 | +[default7]: iteration 2303/ 3814 | consumed samples: 1179136 | consumed tokens: 2414870528 | elapsed time per iteration (s): 5.45 | learning rate: 4.619E-04 | global batch size: 512 | lm loss: 3.231305E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.008 | TFLOPs: 71.01 | +[default7]: iteration 2304/ 3814 | consumed samples: 1179648 | consumed tokens: 2415919104 | elapsed time per iteration (s): 5.43 | learning rate: 4.615E-04 | global batch size: 512 | lm loss: 3.259392E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.293 | TFLOPs: 71.22 | +[default7]: iteration 2305/ 3814 | consumed samples: 1180160 | consumed tokens: 2416967680 | elapsed time per iteration (s): 5.44 | learning rate: 4.611E-04 | global batch size: 512 | lm loss: 3.240011E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.073 | TFLOPs: 71.05 | +[default7]: iteration 2306/ 3814 | consumed samples: 1180672 | consumed tokens: 2418016256 | elapsed time per iteration (s): 5.42 | learning rate: 4.607E-04 | global batch size: 512 | lm loss: 3.222399E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.426 | TFLOPs: 71.32 | +[default7]: iteration 2307/ 3814 | consumed samples: 1181184 | consumed tokens: 2419064832 | elapsed time per iteration (s): 5.41 | learning rate: 4.603E-04 | global batch size: 512 | lm loss: 3.243534E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.579 | TFLOPs: 71.44 | +[default7]: iteration 2308/ 3814 | consumed samples: 1181696 | consumed tokens: 2420113408 | elapsed time per iteration (s): 5.43 | learning rate: 4.599E-04 | global batch size: 512 | lm loss: 3.241933E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.348 | TFLOPs: 71.26 | +[default7]: iteration 2309/ 3814 | consumed samples: 1182208 | consumed tokens: 2421161984 | elapsed time per iteration (s): 5.43 | learning rate: 4.595E-04 | global batch size: 512 | lm loss: 3.216013E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.241 | TFLOPs: 71.18 | +[default7]: iteration 2310/ 3814 | consumed samples: 1182720 | consumed tokens: 2422210560 | elapsed time per iteration (s): 5.43 | learning rate: 4.591E-04 | global batch size: 512 | lm loss: 3.203743E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.281 | TFLOPs: 71.21 | +[default7]: iteration 2311/ 3814 | consumed samples: 1183232 | consumed tokens: 2423259136 | elapsed time per iteration (s): 5.47 | learning rate: 4.587E-04 | global batch size: 512 | lm loss: 3.244771E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.684 | TFLOPs: 70.76 | +[default7]: iteration 2312/ 3814 | consumed samples: 1183744 | consumed tokens: 2424307712 | elapsed time per iteration (s): 5.41 | learning rate: 4.583E-04 | global batch size: 512 | lm loss: 3.222060E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.573 | TFLOPs: 71.43 | +[default7]: iteration 2313/ 3814 | consumed samples: 1184256 | consumed tokens: 2425356288 | elapsed time per iteration (s): 5.40 | learning rate: 4.579E-04 | global batch size: 512 | lm loss: 3.228935E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.775 | TFLOPs: 71.58 | +[default7]: iteration 2314/ 3814 | consumed samples: 1184768 | consumed tokens: 2426404864 | elapsed time per iteration (s): 5.40 | learning rate: 4.575E-04 | global batch size: 512 | lm loss: 3.218138E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.875 | TFLOPs: 71.66 | +[default7]: iteration 2315/ 3814 | consumed samples: 1185280 | consumed tokens: 2427453440 | elapsed time per iteration (s): 5.40 | learning rate: 4.571E-04 | global batch size: 512 | lm loss: 3.225299E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.823 | TFLOPs: 71.62 | +[default7]: iteration 2316/ 3814 | consumed samples: 1185792 | consumed tokens: 2428502016 | elapsed time per iteration (s): 5.41 | learning rate: 4.567E-04 | global batch size: 512 | lm loss: 3.195254E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.566 | TFLOPs: 71.43 | +[default7]: iteration 2317/ 3814 | consumed samples: 1186304 | consumed tokens: 2429550592 | elapsed time per iteration (s): 5.44 | learning rate: 4.563E-04 | global batch size: 512 | lm loss: 3.198433E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.181 | TFLOPs: 71.14 | +[default7]: iteration 2318/ 3814 | consumed samples: 1186816 | consumed tokens: 2430599168 | elapsed time per iteration (s): 5.39 | learning rate: 4.559E-04 | global batch size: 512 | lm loss: 3.238651E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.978 | TFLOPs: 71.74 | +[default7]: iteration 2319/ 3814 | consumed samples: 1187328 | consumed tokens: 2431647744 | elapsed time per iteration (s): 5.42 | learning rate: 4.555E-04 | global batch size: 512 | lm loss: 3.249480E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.474 | TFLOPs: 71.36 | +[default7]: iteration 2320/ 3814 | consumed samples: 1187840 | consumed tokens: 2432696320 | elapsed time per iteration (s): 5.42 | learning rate: 4.551E-04 | global batch size: 512 | lm loss: 3.222805E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.526 | TFLOPs: 71.40 | +[default7]: iteration 2321/ 3814 | consumed samples: 1188352 | consumed tokens: 2433744896 | elapsed time per iteration (s): 5.41 | learning rate: 4.547E-04 | global batch size: 512 | lm loss: 3.247752E+00 | loss scale: 65536.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.659 | TFLOPs: 71.50 | +[default7]: iteration 2322/ 3814 | consumed samples: 1188864 | consumed tokens: 2434793472 | elapsed time per iteration (s): 5.43 | learning rate: 4.543E-04 | global batch size: 512 | lm loss: 3.223851E+00 | loss scale: 65536.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.291 | TFLOPs: 71.22 | +[default7]: iteration 2323/ 3814 | consumed samples: 1189376 | consumed tokens: 2435842048 | elapsed time per iteration (s): 5.41 | learning rate: 4.539E-04 | global batch size: 512 | lm loss: 3.239992E+00 | loss scale: 65536.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.609 | TFLOPs: 71.46 | +[default7]: iteration 2324/ 3814 | consumed samples: 1189888 | consumed tokens: 2436890624 | elapsed time per iteration (s): 5.43 | learning rate: 4.535E-04 | global batch size: 512 | lm loss: 3.231379E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.239 | TFLOPs: 71.18 | +[default7]: iteration 2325/ 3814 | consumed samples: 1190400 | consumed tokens: 2437939200 | elapsed time per iteration (s): 5.43 | learning rate: 4.531E-04 | global batch size: 512 | lm loss: 3.239602E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.247 | TFLOPs: 71.19 | +[default7]: iteration 2326/ 3814 | consumed samples: 1190912 | consumed tokens: 2438987776 | elapsed time per iteration (s): 5.44 | learning rate: 4.527E-04 | global batch size: 512 | lm loss: 3.218010E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.120 | TFLOPs: 71.09 | +[default7]: iteration 2327/ 3814 | consumed samples: 1191424 | consumed tokens: 2440036352 | elapsed time per iteration (s): 5.43 | learning rate: 4.523E-04 | global batch size: 512 | lm loss: 3.191828E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.288 | TFLOPs: 71.22 | +[default7]: iteration 2328/ 3814 | consumed samples: 1191936 | consumed tokens: 2441084928 | elapsed time per iteration (s): 5.42 | learning rate: 4.519E-04 | global batch size: 512 | lm loss: 3.218655E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.416 | TFLOPs: 71.31 | +[default7]: iteration 2329/ 3814 | consumed samples: 1192448 | consumed tokens: 2442133504 | elapsed time per iteration (s): 5.42 | learning rate: 4.515E-04 | global batch size: 512 | lm loss: 3.234799E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.535 | TFLOPs: 71.40 | +[default7]: iteration 2330/ 3814 | consumed samples: 1192960 | consumed tokens: 2443182080 | elapsed time per iteration (s): 5.43 | learning rate: 4.511E-04 | global batch size: 512 | lm loss: 3.226278E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 2331/ 3814 | consumed samples: 1193472 | consumed tokens: 2444230656 | elapsed time per iteration (s): 5.40 | learning rate: 4.507E-04 | global batch size: 512 | lm loss: 3.219640E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.736 | TFLOPs: 71.56 | +[default7]: iteration 2332/ 3814 | consumed samples: 1193984 | consumed tokens: 2445279232 | elapsed time per iteration (s): 5.42 | learning rate: 4.503E-04 | global batch size: 512 | lm loss: 3.232169E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.545 | TFLOPs: 71.41 | +[default7]: iteration 2333/ 3814 | consumed samples: 1194496 | consumed tokens: 2446327808 | elapsed time per iteration (s): 5.46 | learning rate: 4.499E-04 | global batch size: 512 | lm loss: 3.232600E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.769 | TFLOPs: 70.82 | +[default7]: iteration 2334/ 3814 | consumed samples: 1195008 | consumed tokens: 2447376384 | elapsed time per iteration (s): 5.43 | learning rate: 4.495E-04 | global batch size: 512 | lm loss: 3.215081E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.290 | TFLOPs: 71.22 | +[default7]: iteration 2335/ 3814 | consumed samples: 1195520 | consumed tokens: 2448424960 | elapsed time per iteration (s): 5.43 | learning rate: 4.491E-04 | global batch size: 512 | lm loss: 3.225840E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.358 | TFLOPs: 71.27 | +[default7]: iteration 2336/ 3814 | consumed samples: 1196032 | consumed tokens: 2449473536 | elapsed time per iteration (s): 5.43 | learning rate: 4.487E-04 | global batch size: 512 | lm loss: 3.237561E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.235 | TFLOPs: 71.18 | +[default7]: iteration 2337/ 3814 | consumed samples: 1196544 | consumed tokens: 2450522112 | elapsed time per iteration (s): 5.42 | learning rate: 4.483E-04 | global batch size: 512 | lm loss: 3.231960E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.515 | TFLOPs: 71.39 | +[default7]: iteration 2338/ 3814 | consumed samples: 1197056 | consumed tokens: 2451570688 | elapsed time per iteration (s): 5.42 | learning rate: 4.479E-04 | global batch size: 512 | lm loss: 3.216629E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.519 | TFLOPs: 71.39 | +[default7]: iteration 2339/ 3814 | consumed samples: 1197568 | consumed tokens: 2452619264 | elapsed time per iteration (s): 5.40 | learning rate: 4.475E-04 | global batch size: 512 | lm loss: 3.215462E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 2340/ 3814 | consumed samples: 1198080 | consumed tokens: 2453667840 | elapsed time per iteration (s): 5.43 | learning rate: 4.471E-04 | global batch size: 512 | lm loss: 3.252202E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.252 | TFLOPs: 71.19 | +[default7]: iteration 2341/ 3814 | consumed samples: 1198592 | consumed tokens: 2454716416 | elapsed time per iteration (s): 5.41 | learning rate: 4.467E-04 | global batch size: 512 | lm loss: 3.225683E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 2342/ 3814 | consumed samples: 1199104 | consumed tokens: 2455764992 | elapsed time per iteration (s): 5.43 | learning rate: 4.463E-04 | global batch size: 512 | lm loss: 3.192531E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.344 | TFLOPs: 71.26 | +[default7]: iteration 2343/ 3814 | consumed samples: 1199616 | consumed tokens: 2456813568 | elapsed time per iteration (s): 5.43 | learning rate: 4.459E-04 | global batch size: 512 | lm loss: 3.246989E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.242 | TFLOPs: 71.18 | +[default7]: iteration 2344/ 3814 | consumed samples: 1200128 | consumed tokens: 2457862144 | elapsed time per iteration (s): 5.41 | learning rate: 4.455E-04 | global batch size: 512 | lm loss: 3.209486E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2345/ 3814 | consumed samples: 1200640 | consumed tokens: 2458910720 | elapsed time per iteration (s): 5.41 | learning rate: 4.451E-04 | global batch size: 512 | lm loss: 3.211091E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.624 | TFLOPs: 71.47 | +[default7]: iteration 2346/ 3814 | consumed samples: 1201152 | consumed tokens: 2459959296 | elapsed time per iteration (s): 5.40 | learning rate: 4.447E-04 | global batch size: 512 | lm loss: 3.236810E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.750 | TFLOPs: 71.57 | +[default7]: iteration 2347/ 3814 | consumed samples: 1201664 | consumed tokens: 2461007872 | elapsed time per iteration (s): 5.41 | learning rate: 4.443E-04 | global batch size: 512 | lm loss: 3.207556E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.641 | TFLOPs: 71.48 | +[default7]: iteration 2348/ 3814 | consumed samples: 1202176 | consumed tokens: 2462056448 | elapsed time per iteration (s): 5.42 | learning rate: 4.439E-04 | global batch size: 512 | lm loss: 3.233301E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.384 | TFLOPs: 71.29 | +[default7]: iteration 2349/ 3814 | consumed samples: 1202688 | consumed tokens: 2463105024 | elapsed time per iteration (s): 5.44 | learning rate: 4.435E-04 | global batch size: 512 | lm loss: 3.212980E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.202 | TFLOPs: 71.15 | +[default7]: iteration 2350/ 3814 | consumed samples: 1203200 | consumed tokens: 2464153600 | elapsed time per iteration (s): 5.42 | learning rate: 4.431E-04 | global batch size: 512 | lm loss: 3.212995E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.435 | TFLOPs: 71.33 | +[default7]: iteration 2351/ 3814 | consumed samples: 1203712 | consumed tokens: 2465202176 | elapsed time per iteration (s): 5.41 | learning rate: 4.427E-04 | global batch size: 512 | lm loss: 3.199907E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.627 | TFLOPs: 71.47 | +[default7]: iteration 2352/ 3814 | consumed samples: 1204224 | consumed tokens: 2466250752 | elapsed time per iteration (s): 5.41 | learning rate: 4.423E-04 | global batch size: 512 | lm loss: 3.228657E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 2353/ 3814 | consumed samples: 1204736 | consumed tokens: 2467299328 | elapsed time per iteration (s): 5.42 | learning rate: 4.419E-04 | global batch size: 512 | lm loss: 3.192926E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.520 | TFLOPs: 71.39 | +[default7]: iteration 2354/ 3814 | consumed samples: 1205248 | consumed tokens: 2468347904 | elapsed time per iteration (s): 5.41 | learning rate: 4.415E-04 | global batch size: 512 | lm loss: 3.217411E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.47 | +[default7]: iteration 2355/ 3814 | consumed samples: 1205760 | consumed tokens: 2469396480 | elapsed time per iteration (s): 5.42 | learning rate: 4.412E-04 | global batch size: 512 | lm loss: 3.220823E+00 | loss scale: 65536.0 | grad norm: 0.094 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.540 | TFLOPs: 71.41 | +[default7]: iteration 2356/ 3814 | consumed samples: 1206272 | consumed tokens: 2470445056 | elapsed time per iteration (s): 5.44 | learning rate: 4.408E-04 | global batch size: 512 | lm loss: 3.195851E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.171 | TFLOPs: 71.13 | +[default7]: iteration 2357/ 3814 | consumed samples: 1206784 | consumed tokens: 2471493632 | elapsed time per iteration (s): 5.42 | learning rate: 4.404E-04 | global batch size: 512 | lm loss: 3.232284E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 2358/ 3814 | consumed samples: 1207296 | consumed tokens: 2472542208 | elapsed time per iteration (s): 5.43 | learning rate: 4.400E-04 | global batch size: 512 | lm loss: 3.234360E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.348 | TFLOPs: 71.26 | +[default7]: iteration 2359/ 3814 | consumed samples: 1207808 | consumed tokens: 2473590784 | elapsed time per iteration (s): 5.42 | learning rate: 4.396E-04 | global batch size: 512 | lm loss: 3.205610E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.493 | TFLOPs: 71.37 | +[default7]: iteration 2360/ 3814 | consumed samples: 1208320 | consumed tokens: 2474639360 | elapsed time per iteration (s): 5.41 | learning rate: 4.392E-04 | global batch size: 512 | lm loss: 3.217307E+00 | loss scale: 65536.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.610 | TFLOPs: 71.46 | +[default7]: iteration 2361/ 3814 | consumed samples: 1208832 | consumed tokens: 2475687936 | elapsed time per iteration (s): 5.43 | learning rate: 4.388E-04 | global batch size: 512 | lm loss: 3.238929E+00 | loss scale: 65536.0 | grad norm: 0.143 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.283 | TFLOPs: 71.21 | +[default7]: iteration 2362/ 3814 | consumed samples: 1209344 | consumed tokens: 2476736512 | elapsed time per iteration (s): 5.42 | learning rate: 4.384E-04 | global batch size: 512 | lm loss: 3.222830E+00 | loss scale: 65536.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.390 | TFLOPs: 71.29 | +[default7]: iteration 2363/ 3814 | consumed samples: 1209856 | consumed tokens: 2477785088 | elapsed time per iteration (s): 5.43 | learning rate: 4.380E-04 | global batch size: 512 | lm loss: 3.222082E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 2364/ 3814 | consumed samples: 1210368 | consumed tokens: 2478833664 | elapsed time per iteration (s): 5.44 | learning rate: 4.376E-04 | global batch size: 512 | lm loss: 3.230476E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.154 | TFLOPs: 71.12 | +[default7]: iteration 2365/ 3814 | consumed samples: 1210880 | consumed tokens: 2479882240 | elapsed time per iteration (s): 5.42 | learning rate: 4.372E-04 | global batch size: 512 | lm loss: 3.222894E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.384 | TFLOPs: 71.29 | +[default7]: iteration 2366/ 3814 | consumed samples: 1211392 | consumed tokens: 2480930816 | elapsed time per iteration (s): 5.42 | learning rate: 4.368E-04 | global batch size: 512 | lm loss: 3.234405E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.437 | TFLOPs: 71.33 | +[default7]: iteration 2367/ 3814 | consumed samples: 1211904 | consumed tokens: 2481979392 | elapsed time per iteration (s): 5.42 | learning rate: 4.364E-04 | global batch size: 512 | lm loss: 3.226740E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.474 | TFLOPs: 71.36 | +[default7]: iteration 2368/ 3814 | consumed samples: 1212416 | consumed tokens: 2483027968 | elapsed time per iteration (s): 5.41 | learning rate: 4.360E-04 | global batch size: 512 | lm loss: 3.195904E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.700 | TFLOPs: 71.53 | +[default7]: iteration 2369/ 3814 | consumed samples: 1212928 | consumed tokens: 2484076544 | elapsed time per iteration (s): 5.43 | learning rate: 4.356E-04 | global batch size: 512 | lm loss: 3.200521E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.307 | TFLOPs: 71.23 | +[default7]: iteration 2370/ 3814 | consumed samples: 1213440 | consumed tokens: 2485125120 | elapsed time per iteration (s): 5.43 | learning rate: 4.352E-04 | global batch size: 512 | lm loss: 3.214697E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.282 | TFLOPs: 71.21 | +[default7]: iteration 2371/ 3814 | consumed samples: 1213952 | consumed tokens: 2486173696 | elapsed time per iteration (s): 5.42 | learning rate: 4.348E-04 | global batch size: 512 | lm loss: 3.204458E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.427 | TFLOPs: 71.32 | +[default7]: iteration 2372/ 3814 | consumed samples: 1214464 | consumed tokens: 2487222272 | elapsed time per iteration (s): 5.42 | learning rate: 4.344E-04 | global batch size: 512 | lm loss: 3.231970E+00 | loss scale: 65536.0 | grad norm: 0.093 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.405 | TFLOPs: 71.31 | +[default7]: iteration 2373/ 3814 | consumed samples: 1214976 | consumed tokens: 2488270848 | elapsed time per iteration (s): 5.43 | learning rate: 4.340E-04 | global batch size: 512 | lm loss: 3.244973E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.239 | TFLOPs: 71.18 | +[default7]: iteration 2374/ 3814 | consumed samples: 1215488 | consumed tokens: 2489319424 | elapsed time per iteration (s): 5.43 | learning rate: 4.336E-04 | global batch size: 512 | lm loss: 3.228082E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.326 | TFLOPs: 71.25 | +[default7]: iteration 2375/ 3814 | consumed samples: 1216000 | consumed tokens: 2490368000 | elapsed time per iteration (s): 5.41 | learning rate: 4.332E-04 | global batch size: 512 | lm loss: 3.234594E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.665 | TFLOPs: 71.50 | +[default7]: iteration 2376/ 3814 | consumed samples: 1216512 | consumed tokens: 2491416576 | elapsed time per iteration (s): 5.41 | learning rate: 4.328E-04 | global batch size: 512 | lm loss: 3.193944E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.614 | TFLOPs: 71.46 | +[default7]: iteration 2377/ 3814 | consumed samples: 1217024 | consumed tokens: 2492465152 | elapsed time per iteration (s): 5.41 | learning rate: 4.324E-04 | global batch size: 512 | lm loss: 3.245633E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2378/ 3814 | consumed samples: 1217536 | consumed tokens: 2493513728 | elapsed time per iteration (s): 5.40 | learning rate: 4.321E-04 | global batch size: 512 | lm loss: 3.178140E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.755 | TFLOPs: 71.57 | +[default7]: iteration 2379/ 3814 | consumed samples: 1218048 | consumed tokens: 2494562304 | elapsed time per iteration (s): 5.41 | learning rate: 4.317E-04 | global batch size: 512 | lm loss: 3.221102E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.686 | TFLOPs: 71.52 | +[default7]: iteration 2380/ 3814 | consumed samples: 1218560 | consumed tokens: 2495610880 | elapsed time per iteration (s): 5.43 | learning rate: 4.313E-04 | global batch size: 512 | lm loss: 3.220194E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.224 | TFLOPs: 71.17 | +[default7]: iteration 2381/ 3814 | consumed samples: 1219072 | consumed tokens: 2496659456 | elapsed time per iteration (s): 5.43 | learning rate: 4.309E-04 | global batch size: 512 | lm loss: 3.224539E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 2382/ 3814 | consumed samples: 1219584 | consumed tokens: 2497708032 | elapsed time per iteration (s): 5.42 | learning rate: 4.305E-04 | global batch size: 512 | lm loss: 3.205494E+00 | loss scale: 65536.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 2383/ 3814 | consumed samples: 1220096 | consumed tokens: 2498756608 | elapsed time per iteration (s): 5.42 | learning rate: 4.301E-04 | global batch size: 512 | lm loss: 3.190068E+00 | loss scale: 65536.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.441 | TFLOPs: 71.33 | +[default7]: iteration 2384/ 3814 | consumed samples: 1220608 | consumed tokens: 2499805184 | elapsed time per iteration (s): 5.42 | learning rate: 4.297E-04 | global batch size: 512 | lm loss: 3.204849E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.484 | TFLOPs: 71.36 | +[default7]: iteration 2385/ 3814 | consumed samples: 1221120 | consumed tokens: 2500853760 | elapsed time per iteration (s): 5.42 | learning rate: 4.293E-04 | global batch size: 512 | lm loss: 3.227797E+00 | loss scale: 65536.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.463 | TFLOPs: 71.35 | +[default7]: iteration 2386/ 3814 | consumed samples: 1221632 | consumed tokens: 2501902336 | elapsed time per iteration (s): 5.43 | learning rate: 4.289E-04 | global batch size: 512 | lm loss: 3.190828E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.349 | TFLOPs: 71.26 | +[default7]: iteration 2387/ 3814 | consumed samples: 1222144 | consumed tokens: 2502950912 | elapsed time per iteration (s): 5.42 | learning rate: 4.285E-04 | global batch size: 512 | lm loss: 3.203974E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.439 | TFLOPs: 71.33 | +[default7]: iteration 2388/ 3814 | consumed samples: 1222656 | consumed tokens: 2503999488 | elapsed time per iteration (s): 5.42 | learning rate: 4.281E-04 | global batch size: 512 | lm loss: 3.222584E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.454 | TFLOPs: 71.34 | +[default7]: iteration 2389/ 3814 | consumed samples: 1223168 | consumed tokens: 2505048064 | elapsed time per iteration (s): 5.43 | learning rate: 4.277E-04 | global batch size: 512 | lm loss: 3.234601E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.339 | TFLOPs: 71.25 | +[default7]: iteration 2390/ 3814 | consumed samples: 1223680 | consumed tokens: 2506096640 | elapsed time per iteration (s): 5.43 | learning rate: 4.273E-04 | global batch size: 512 | lm loss: 3.231538E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 2391/ 3814 | consumed samples: 1224192 | consumed tokens: 2507145216 | elapsed time per iteration (s): 5.43 | learning rate: 4.269E-04 | global batch size: 512 | lm loss: 3.229971E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.272 | TFLOPs: 71.20 | +[default7]: iteration 2392/ 3814 | consumed samples: 1224704 | consumed tokens: 2508193792 | elapsed time per iteration (s): 5.42 | learning rate: 4.265E-04 | global batch size: 512 | lm loss: 3.218326E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.480 | TFLOPs: 71.36 | +[default7]: iteration 2393/ 3814 | consumed samples: 1225216 | consumed tokens: 2509242368 | elapsed time per iteration (s): 5.42 | learning rate: 4.261E-04 | global batch size: 512 | lm loss: 3.205569E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 2394/ 3814 | consumed samples: 1225728 | consumed tokens: 2510290944 | elapsed time per iteration (s): 5.40 | learning rate: 4.257E-04 | global batch size: 512 | lm loss: 3.223016E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.863 | TFLOPs: 71.65 | +[default7]: iteration 2395/ 3814 | consumed samples: 1226240 | consumed tokens: 2511339520 | elapsed time per iteration (s): 5.40 | learning rate: 4.254E-04 | global batch size: 512 | lm loss: 3.179619E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.769 | TFLOPs: 71.58 | +[default7]: iteration 2396/ 3814 | consumed samples: 1226752 | consumed tokens: 2512388096 | elapsed time per iteration (s): 5.41 | learning rate: 4.250E-04 | global batch size: 512 | lm loss: 3.254339E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.655 | TFLOPs: 71.49 | +[default7]: iteration 2397/ 3814 | consumed samples: 1227264 | consumed tokens: 2513436672 | elapsed time per iteration (s): 5.40 | learning rate: 4.246E-04 | global batch size: 512 | lm loss: 3.228371E+00 | loss scale: 65536.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.832 | TFLOPs: 71.63 | +[default7]: iteration 2398/ 3814 | consumed samples: 1227776 | consumed tokens: 2514485248 | elapsed time per iteration (s): 5.40 | learning rate: 4.242E-04 | global batch size: 512 | lm loss: 3.198688E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.826 | TFLOPs: 71.62 | +[default7]: iteration 2399/ 3814 | consumed samples: 1228288 | consumed tokens: 2515533824 | elapsed time per iteration (s): 5.41 | learning rate: 4.238E-04 | global batch size: 512 | lm loss: 3.210399E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.601 | TFLOPs: 71.45 | +[default7]: iteration 2400/ 3814 | consumed samples: 1228800 | consumed tokens: 2516582400 | elapsed time per iteration (s): 5.45 | learning rate: 4.234E-04 | global batch size: 512 | lm loss: 3.204329E+00 | loss scale: 65536.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.977 | TFLOPs: 70.98 | +[default7]: iteration 2401/ 3814 | consumed samples: 1229312 | consumed tokens: 2517630976 | elapsed time per iteration (s): 5.42 | learning rate: 4.230E-04 | global batch size: 512 | lm loss: 3.226104E+00 | loss scale: 65536.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 2402/ 3814 | consumed samples: 1229824 | consumed tokens: 2518679552 | elapsed time per iteration (s): 5.42 | learning rate: 4.226E-04 | global batch size: 512 | lm loss: 3.230648E+00 | loss scale: 65536.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.534 | TFLOPs: 71.40 | +[default7]: iteration 2403/ 3814 | consumed samples: 1230336 | consumed tokens: 2519728128 | elapsed time per iteration (s): 5.40 | learning rate: 4.222E-04 | global batch size: 512 | lm loss: 3.232812E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.751 | TFLOPs: 71.57 | +[default7]: iteration 2404/ 3814 | consumed samples: 1230848 | consumed tokens: 2520776704 | elapsed time per iteration (s): 5.43 | learning rate: 4.218E-04 | global batch size: 512 | lm loss: 3.214753E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 2405/ 3814 | consumed samples: 1231360 | consumed tokens: 2521825280 | elapsed time per iteration (s): 5.42 | learning rate: 4.214E-04 | global batch size: 512 | lm loss: 3.225497E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 2406/ 3814 | consumed samples: 1231872 | consumed tokens: 2522873856 | elapsed time per iteration (s): 5.43 | learning rate: 4.210E-04 | global batch size: 512 | lm loss: 3.206978E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.286 | TFLOPs: 71.22 | +[default7]: iteration 2407/ 3814 | consumed samples: 1232384 | consumed tokens: 2523922432 | elapsed time per iteration (s): 5.44 | learning rate: 4.206E-04 | global batch size: 512 | lm loss: 3.198711E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.143 | TFLOPs: 71.11 | +[default7]: iteration 2408/ 3814 | consumed samples: 1232896 | consumed tokens: 2524971008 | elapsed time per iteration (s): 5.43 | learning rate: 4.203E-04 | global batch size: 512 | lm loss: 3.194181E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.281 | TFLOPs: 71.21 | +[default7]: iteration 2409/ 3814 | consumed samples: 1233408 | consumed tokens: 2526019584 | elapsed time per iteration (s): 5.43 | learning rate: 4.199E-04 | global batch size: 512 | lm loss: 3.179685E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.279 | TFLOPs: 71.21 | +[default7]: iteration 2410/ 3814 | consumed samples: 1233920 | consumed tokens: 2527068160 | elapsed time per iteration (s): 5.44 | learning rate: 4.195E-04 | global batch size: 512 | lm loss: 3.184369E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.151 | TFLOPs: 71.11 | +[default7]: iteration 2411/ 3814 | consumed samples: 1234432 | consumed tokens: 2528116736 | elapsed time per iteration (s): 5.46 | learning rate: 4.191E-04 | global batch size: 512 | lm loss: 3.234944E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.838 | TFLOPs: 70.88 | +[default7]: iteration 2412/ 3814 | consumed samples: 1234944 | consumed tokens: 2529165312 | elapsed time per iteration (s): 5.44 | learning rate: 4.187E-04 | global batch size: 512 | lm loss: 3.188578E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.158 | TFLOPs: 71.12 | +[default7]: iteration 2413/ 3814 | consumed samples: 1235456 | consumed tokens: 2530213888 | elapsed time per iteration (s): 5.48 | learning rate: 4.183E-04 | global batch size: 512 | lm loss: 3.197553E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.423 | TFLOPs: 70.56 | +[default7]: iteration 2414/ 3814 | consumed samples: 1235968 | consumed tokens: 2531262464 | elapsed time per iteration (s): 5.45 | learning rate: 4.179E-04 | global batch size: 512 | lm loss: 3.217194E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.013 | TFLOPs: 71.01 | +[default7]: iteration 2415/ 3814 | consumed samples: 1236480 | consumed tokens: 2532311040 | elapsed time per iteration (s): 5.42 | learning rate: 4.175E-04 | global batch size: 512 | lm loss: 3.220537E+00 | loss scale: 65536.0 | grad norm: 0.096 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.490 | TFLOPs: 71.37 | +[default7]: iteration 2416/ 3814 | consumed samples: 1236992 | consumed tokens: 2533359616 | elapsed time per iteration (s): 5.43 | learning rate: 4.171E-04 | global batch size: 512 | lm loss: 3.213655E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.300 | TFLOPs: 71.23 | +[default7]: iteration 2417/ 3814 | consumed samples: 1237504 | consumed tokens: 2534408192 | elapsed time per iteration (s): 5.42 | learning rate: 4.167E-04 | global batch size: 512 | lm loss: 3.237749E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.492 | TFLOPs: 71.37 | +[default7]: iteration 2418/ 3814 | consumed samples: 1238016 | consumed tokens: 2535456768 | elapsed time per iteration (s): 5.44 | learning rate: 4.163E-04 | global batch size: 512 | lm loss: 3.204420E+00 | loss scale: 65536.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.179 | TFLOPs: 71.13 | +[default7]: iteration 2419/ 3814 | consumed samples: 1238528 | consumed tokens: 2536505344 | elapsed time per iteration (s): 5.44 | learning rate: 4.160E-04 | global batch size: 512 | lm loss: 3.179388E+00 | loss scale: 65536.0 | grad norm: 0.135 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.071 | TFLOPs: 71.05 | +[default7]: iteration 2420/ 3814 | consumed samples: 1239040 | consumed tokens: 2537553920 | elapsed time per iteration (s): 5.42 | learning rate: 4.156E-04 | global batch size: 512 | lm loss: 3.208840E+00 | loss scale: 65536.0 | grad norm: 0.131 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.451 | TFLOPs: 71.34 | +[default7]: iteration 2421/ 3814 | consumed samples: 1239552 | consumed tokens: 2538602496 | elapsed time per iteration (s): 5.41 | learning rate: 4.152E-04 | global batch size: 512 | lm loss: 3.192569E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.663 | TFLOPs: 71.50 | +[default7]: iteration 2422/ 3814 | consumed samples: 1240064 | consumed tokens: 2539651072 | elapsed time per iteration (s): 5.41 | learning rate: 4.148E-04 | global batch size: 512 | lm loss: 3.208250E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.562 | TFLOPs: 71.42 | +[default7]: iteration 2423/ 3814 | consumed samples: 1240576 | consumed tokens: 2540699648 | elapsed time per iteration (s): 5.45 | learning rate: 4.144E-04 | global batch size: 512 | lm loss: 3.196578E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.959 | TFLOPs: 70.97 | +[default7]: iteration 2424/ 3814 | consumed samples: 1241088 | consumed tokens: 2541748224 | elapsed time per iteration (s): 5.43 | learning rate: 4.140E-04 | global batch size: 512 | lm loss: 3.190913E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.323 | TFLOPs: 71.24 | +[default7]: iteration 2425/ 3814 | consumed samples: 1241600 | consumed tokens: 2542796800 | elapsed time per iteration (s): 5.43 | learning rate: 4.136E-04 | global batch size: 512 | lm loss: 3.204650E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 2426/ 3814 | consumed samples: 1242112 | consumed tokens: 2543845376 | elapsed time per iteration (s): 5.43 | learning rate: 4.132E-04 | global batch size: 512 | lm loss: 3.216729E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.327 | TFLOPs: 71.25 | +[default7]: iteration 2427/ 3814 | consumed samples: 1242624 | consumed tokens: 2544893952 | elapsed time per iteration (s): 5.47 | learning rate: 4.128E-04 | global batch size: 512 | lm loss: 3.192245E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.632 | TFLOPs: 70.72 | +[default7]: iteration 2428/ 3814 | consumed samples: 1243136 | consumed tokens: 2545942528 | elapsed time per iteration (s): 5.43 | learning rate: 4.124E-04 | global batch size: 512 | lm loss: 3.211903E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.307 | TFLOPs: 71.23 | +[default7]: iteration 2429/ 3814 | consumed samples: 1243648 | consumed tokens: 2546991104 | elapsed time per iteration (s): 5.44 | learning rate: 4.121E-04 | global batch size: 512 | lm loss: 3.217829E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.166 | TFLOPs: 71.12 | +[default7]: iteration 2430/ 3814 | consumed samples: 1244160 | consumed tokens: 2548039680 | elapsed time per iteration (s): 5.45 | learning rate: 4.117E-04 | global batch size: 512 | lm loss: 3.166599E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.920 | TFLOPs: 70.94 | +[default7]: iteration 2431/ 3814 | consumed samples: 1244672 | consumed tokens: 2549088256 | elapsed time per iteration (s): 5.42 | learning rate: 4.113E-04 | global batch size: 512 | lm loss: 3.202449E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.436 | TFLOPs: 71.33 | +[default7]: iteration 2432/ 3814 | consumed samples: 1245184 | consumed tokens: 2550136832 | elapsed time per iteration (s): 5.43 | learning rate: 4.109E-04 | global batch size: 512 | lm loss: 3.223893E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.356 | TFLOPs: 71.27 | +[default7]: iteration 2433/ 3814 | consumed samples: 1245696 | consumed tokens: 2551185408 | elapsed time per iteration (s): 5.42 | learning rate: 4.105E-04 | global batch size: 512 | lm loss: 3.204863E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.422 | TFLOPs: 71.32 | +[default7]: iteration 2434/ 3814 | consumed samples: 1246208 | consumed tokens: 2552233984 | elapsed time per iteration (s): 5.43 | learning rate: 4.101E-04 | global batch size: 512 | lm loss: 3.203490E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.312 | TFLOPs: 71.24 | +[default7]: iteration 2435/ 3814 | consumed samples: 1246720 | consumed tokens: 2553282560 | elapsed time per iteration (s): 5.43 | learning rate: 4.097E-04 | global batch size: 512 | lm loss: 3.203454E+00 | loss scale: 65536.0 | grad norm: 0.092 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.216 | TFLOPs: 71.16 | +[default7]: iteration 2436/ 3814 | consumed samples: 1247232 | consumed tokens: 2554331136 | elapsed time per iteration (s): 5.43 | learning rate: 4.093E-04 | global batch size: 512 | lm loss: 3.218012E+00 | loss scale: 65536.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.329 | TFLOPs: 71.25 | +[default7]: iteration 2437/ 3814 | consumed samples: 1247744 | consumed tokens: 2555379712 | elapsed time per iteration (s): 5.44 | learning rate: 4.089E-04 | global batch size: 512 | lm loss: 3.198446E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.109 | TFLOPs: 71.08 | +[default7]: iteration 2438/ 3814 | consumed samples: 1248256 | consumed tokens: 2556428288 | elapsed time per iteration (s): 5.45 | learning rate: 4.086E-04 | global batch size: 512 | lm loss: 3.200056E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.958 | TFLOPs: 70.97 | +[default7]: iteration 2439/ 3814 | consumed samples: 1248768 | consumed tokens: 2557476864 | elapsed time per iteration (s): 5.42 | learning rate: 4.082E-04 | global batch size: 512 | lm loss: 3.198505E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.437 | TFLOPs: 71.33 | +[default7]: iteration 2440/ 3814 | consumed samples: 1249280 | consumed tokens: 2558525440 | elapsed time per iteration (s): 5.43 | learning rate: 4.078E-04 | global batch size: 512 | lm loss: 3.153147E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.210 | TFLOPs: 71.16 | +[default7]: iteration 2441/ 3814 | consumed samples: 1249792 | consumed tokens: 2559574016 | elapsed time per iteration (s): 5.42 | learning rate: 4.074E-04 | global batch size: 512 | lm loss: 3.199797E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 2442/ 3814 | consumed samples: 1250304 | consumed tokens: 2560622592 | elapsed time per iteration (s): 5.42 | learning rate: 4.070E-04 | global batch size: 512 | lm loss: 3.194116E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.487 | TFLOPs: 71.37 | +[default7]: iteration 2443/ 3814 | consumed samples: 1250816 | consumed tokens: 2561671168 | elapsed time per iteration (s): 5.40 | learning rate: 4.066E-04 | global batch size: 512 | lm loss: 3.211047E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.780 | TFLOPs: 71.59 | +[default7]: iteration 2444/ 3814 | consumed samples: 1251328 | consumed tokens: 2562719744 | elapsed time per iteration (s): 5.41 | learning rate: 4.062E-04 | global batch size: 512 | lm loss: 3.174920E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.556 | TFLOPs: 71.42 | +[default7]: iteration 2445/ 3814 | consumed samples: 1251840 | consumed tokens: 2563768320 | elapsed time per iteration (s): 5.42 | learning rate: 4.058E-04 | global batch size: 512 | lm loss: 3.203728E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.384 | TFLOPs: 71.29 | +[default7]: iteration 2446/ 3814 | consumed samples: 1252352 | consumed tokens: 2564816896 | elapsed time per iteration (s): 5.42 | learning rate: 4.055E-04 | global batch size: 512 | lm loss: 3.207093E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.476 | TFLOPs: 71.36 | +[default7]: iteration 2447/ 3814 | consumed samples: 1252864 | consumed tokens: 2565865472 | elapsed time per iteration (s): 5.42 | learning rate: 4.051E-04 | global batch size: 512 | lm loss: 3.174467E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.530 | TFLOPs: 71.40 | +[default7]: iteration 2448/ 3814 | consumed samples: 1253376 | consumed tokens: 2566914048 | elapsed time per iteration (s): 5.43 | learning rate: 4.047E-04 | global batch size: 512 | lm loss: 3.208016E+00 | loss scale: 65536.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.295 | TFLOPs: 71.22 | +[default7]: iteration 2449/ 3814 | consumed samples: 1253888 | consumed tokens: 2567962624 | elapsed time per iteration (s): 5.43 | learning rate: 4.043E-04 | global batch size: 512 | lm loss: 3.179264E+00 | loss scale: 65536.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.222 | TFLOPs: 71.17 | +[default7]: iteration 2450/ 3814 | consumed samples: 1254400 | consumed tokens: 2569011200 | elapsed time per iteration (s): 5.41 | learning rate: 4.039E-04 | global batch size: 512 | lm loss: 3.194724E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 2451/ 3814 | consumed samples: 1254912 | consumed tokens: 2570059776 | elapsed time per iteration (s): 5.40 | learning rate: 4.035E-04 | global batch size: 512 | lm loss: 3.174982E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 2452/ 3814 | consumed samples: 1255424 | consumed tokens: 2571108352 | elapsed time per iteration (s): 5.40 | learning rate: 4.031E-04 | global batch size: 512 | lm loss: 3.208536E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.833 | TFLOPs: 71.63 | +[default7]: iteration 2453/ 3814 | consumed samples: 1255936 | consumed tokens: 2572156928 | elapsed time per iteration (s): 5.42 | learning rate: 4.028E-04 | global batch size: 512 | lm loss: 3.221304E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.527 | TFLOPs: 71.40 | +[default7]: iteration 2454/ 3814 | consumed samples: 1256448 | consumed tokens: 2573205504 | elapsed time per iteration (s): 5.40 | learning rate: 4.024E-04 | global batch size: 512 | lm loss: 3.189946E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 2455/ 3814 | consumed samples: 1256960 | consumed tokens: 2574254080 | elapsed time per iteration (s): 5.41 | learning rate: 4.020E-04 | global batch size: 512 | lm loss: 3.218165E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.703 | TFLOPs: 71.53 | +[default7]: iteration 2456/ 3814 | consumed samples: 1257472 | consumed tokens: 2575302656 | elapsed time per iteration (s): 5.41 | learning rate: 4.016E-04 | global batch size: 512 | lm loss: 3.187650E+00 | loss scale: 65536.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.609 | TFLOPs: 71.46 | +[default7]: iteration 2457/ 3814 | consumed samples: 1257984 | consumed tokens: 2576351232 | elapsed time per iteration (s): 5.41 | learning rate: 4.012E-04 | global batch size: 512 | lm loss: 3.212691E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.639 | TFLOPs: 71.48 | +[default7]: iteration 2458/ 3814 | consumed samples: 1258496 | consumed tokens: 2577399808 | elapsed time per iteration (s): 5.41 | learning rate: 4.008E-04 | global batch size: 512 | lm loss: 3.170974E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 2459/ 3814 | consumed samples: 1259008 | consumed tokens: 2578448384 | elapsed time per iteration (s): 5.42 | learning rate: 4.004E-04 | global batch size: 512 | lm loss: 3.204574E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.486 | TFLOPs: 71.37 | +[default7]: iteration 2460/ 3814 | consumed samples: 1259520 | consumed tokens: 2579496960 | elapsed time per iteration (s): 5.43 | learning rate: 4.000E-04 | global batch size: 512 | lm loss: 3.179749E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.224 | TFLOPs: 71.17 | +[default7]: iteration 2461/ 3814 | consumed samples: 1260032 | consumed tokens: 2580545536 | elapsed time per iteration (s): 5.43 | learning rate: 3.997E-04 | global batch size: 512 | lm loss: 3.211708E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.269 | TFLOPs: 71.20 | +[default7]: iteration 2462/ 3814 | consumed samples: 1260544 | consumed tokens: 2581594112 | elapsed time per iteration (s): 5.43 | learning rate: 3.993E-04 | global batch size: 512 | lm loss: 3.190940E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.374 | TFLOPs: 71.28 | +[default7]: iteration 2463/ 3814 | consumed samples: 1261056 | consumed tokens: 2582642688 | elapsed time per iteration (s): 5.42 | learning rate: 3.989E-04 | global batch size: 512 | lm loss: 3.179209E+00 | loss scale: 65536.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.393 | TFLOPs: 71.30 | +[default7]: iteration 2464/ 3814 | consumed samples: 1261568 | consumed tokens: 2583691264 | elapsed time per iteration (s): 5.40 | learning rate: 3.985E-04 | global batch size: 512 | lm loss: 3.197462E+00 | loss scale: 65536.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 2465/ 3814 | consumed samples: 1262080 | consumed tokens: 2584739840 | elapsed time per iteration (s): 5.39 | learning rate: 3.981E-04 | global batch size: 512 | lm loss: 3.208948E+00 | loss scale: 65536.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.000 | TFLOPs: 71.75 | +[default7]: iteration 2466/ 3814 | consumed samples: 1262592 | consumed tokens: 2585788416 | elapsed time per iteration (s): 5.40 | learning rate: 3.977E-04 | global batch size: 512 | lm loss: 3.186507E+00 | loss scale: 65536.0 | grad norm: 0.095 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.738 | TFLOPs: 71.56 | +[default7]: iteration 2467/ 3814 | consumed samples: 1263104 | consumed tokens: 2586836992 | elapsed time per iteration (s): 5.44 | learning rate: 3.974E-04 | global batch size: 512 | lm loss: 3.189985E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.178 | TFLOPs: 71.13 | +[default7]: iteration 2468/ 3814 | consumed samples: 1263616 | consumed tokens: 2587885568 | elapsed time per iteration (s): 5.42 | learning rate: 3.970E-04 | global batch size: 512 | lm loss: 3.199416E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.492 | TFLOPs: 71.37 | +[default7]: iteration 2469/ 3814 | consumed samples: 1264128 | consumed tokens: 2588934144 | elapsed time per iteration (s): 5.42 | learning rate: 3.966E-04 | global batch size: 512 | lm loss: 3.205070E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.415 | TFLOPs: 71.31 | +[default7]: iteration 2470/ 3814 | consumed samples: 1264640 | consumed tokens: 2589982720 | elapsed time per iteration (s): 5.41 | learning rate: 3.962E-04 | global batch size: 512 | lm loss: 3.227124E+00 | loss scale: 65536.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.656 | TFLOPs: 71.49 | +[default7]: iteration 2471/ 3814 | consumed samples: 1265152 | consumed tokens: 2591031296 | elapsed time per iteration (s): 5.42 | learning rate: 3.958E-04 | global batch size: 512 | lm loss: 3.206863E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.477 | TFLOPs: 71.36 | +[default7]: iteration 2472/ 3814 | consumed samples: 1265664 | consumed tokens: 2592079872 | elapsed time per iteration (s): 5.42 | learning rate: 3.954E-04 | global batch size: 512 | lm loss: 3.196338E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.470 | TFLOPs: 71.35 | +[default7]: iteration 2473/ 3814 | consumed samples: 1266176 | consumed tokens: 2593128448 | elapsed time per iteration (s): 5.41 | learning rate: 3.950E-04 | global batch size: 512 | lm loss: 3.181798E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.676 | TFLOPs: 71.51 | +[default7]: iteration 2474/ 3814 | consumed samples: 1266688 | consumed tokens: 2594177024 | elapsed time per iteration (s): 5.41 | learning rate: 3.947E-04 | global batch size: 512 | lm loss: 3.169253E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.617 | TFLOPs: 71.46 | +[default7]: iteration 2475/ 3814 | consumed samples: 1267200 | consumed tokens: 2595225600 | elapsed time per iteration (s): 5.43 | learning rate: 3.943E-04 | global batch size: 512 | lm loss: 3.201644E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.206 | TFLOPs: 71.15 | +[default7]: iteration 2476/ 3814 | consumed samples: 1267712 | consumed tokens: 2596274176 | elapsed time per iteration (s): 5.42 | learning rate: 3.939E-04 | global batch size: 512 | lm loss: 3.181870E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 2477/ 3814 | consumed samples: 1268224 | consumed tokens: 2597322752 | elapsed time per iteration (s): 5.40 | learning rate: 3.935E-04 | global batch size: 512 | lm loss: 3.203845E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.863 | TFLOPs: 71.65 | +[default7]: iteration 2478/ 3814 | consumed samples: 1268736 | consumed tokens: 2598371328 | elapsed time per iteration (s): 5.41 | learning rate: 3.931E-04 | global batch size: 512 | lm loss: 3.212695E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.574 | TFLOPs: 71.43 | +[default7]: iteration 2479/ 3814 | consumed samples: 1269248 | consumed tokens: 2599419904 | elapsed time per iteration (s): 5.44 | learning rate: 3.927E-04 | global batch size: 512 | lm loss: 3.193370E+00 | loss scale: 65536.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.172 | TFLOPs: 71.13 | +[default7]: iteration 2480/ 3814 | consumed samples: 1269760 | consumed tokens: 2600468480 | elapsed time per iteration (s): 5.45 | learning rate: 3.924E-04 | global batch size: 512 | lm loss: 3.195481E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.028 | TFLOPs: 71.02 | +[default7]: iteration 2481/ 3814 | consumed samples: 1270272 | consumed tokens: 2601517056 | elapsed time per iteration (s): 5.42 | learning rate: 3.920E-04 | global batch size: 512 | lm loss: 3.161218E+00 | loss scale: 65536.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.395 | TFLOPs: 71.30 | +[default7]: iteration 2482/ 3814 | consumed samples: 1270784 | consumed tokens: 2602565632 | elapsed time per iteration (s): 5.42 | learning rate: 3.916E-04 | global batch size: 512 | lm loss: 3.170875E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.415 | TFLOPs: 71.31 | +[default7]: iteration 2483/ 3814 | consumed samples: 1271296 | consumed tokens: 2603614208 | elapsed time per iteration (s): 5.41 | learning rate: 3.912E-04 | global batch size: 512 | lm loss: 3.200699E+00 | loss scale: 65536.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.647 | TFLOPs: 71.49 | +[default7]: iteration 2484/ 3814 | consumed samples: 1271808 | consumed tokens: 2604662784 | elapsed time per iteration (s): 5.41 | learning rate: 3.908E-04 | global batch size: 512 | lm loss: 3.187665E+00 | loss scale: 65536.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.723 | TFLOPs: 71.55 | +[default7]: iteration 2485/ 3814 | consumed samples: 1272320 | consumed tokens: 2605711360 | elapsed time per iteration (s): 5.41 | learning rate: 3.904E-04 | global batch size: 512 | lm loss: 3.179887E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 2486/ 3814 | consumed samples: 1272832 | consumed tokens: 2606759936 | elapsed time per iteration (s): 5.42 | learning rate: 3.901E-04 | global batch size: 512 | lm loss: 3.175282E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.437 | TFLOPs: 71.33 | +[default7]: iteration 2487/ 3814 | consumed samples: 1273344 | consumed tokens: 2607808512 | elapsed time per iteration (s): 5.41 | learning rate: 3.897E-04 | global batch size: 512 | lm loss: 3.170747E+00 | loss scale: 65536.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.578 | TFLOPs: 71.44 | +[default7]: iteration 2488/ 3814 | consumed samples: 1273856 | consumed tokens: 2608857088 | elapsed time per iteration (s): 5.41 | learning rate: 3.893E-04 | global batch size: 512 | lm loss: 3.194670E+00 | loss scale: 65536.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 2489/ 3814 | consumed samples: 1274368 | consumed tokens: 2609905664 | elapsed time per iteration (s): 5.41 | learning rate: 3.889E-04 | global batch size: 512 | lm loss: 3.179883E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.638 | TFLOPs: 71.48 | +[default7]: iteration 2490/ 3814 | consumed samples: 1274880 | consumed tokens: 2610954240 | elapsed time per iteration (s): 5.44 | learning rate: 3.885E-04 | global batch size: 512 | lm loss: 3.208250E+00 | loss scale: 65536.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.069 | TFLOPs: 71.05 | +[default7]: iteration 2491/ 3814 | consumed samples: 1275392 | consumed tokens: 2612002816 | elapsed time per iteration (s): 5.41 | learning rate: 3.882E-04 | global batch size: 512 | lm loss: 3.157362E+00 | loss scale: 65536.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 2492/ 3814 | consumed samples: 1275904 | consumed tokens: 2613051392 | elapsed time per iteration (s): 5.41 | learning rate: 3.878E-04 | global batch size: 512 | lm loss: 3.187782E+00 | loss scale: 65536.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.561 | TFLOPs: 71.42 | +[default7]: iteration 2493/ 3814 | consumed samples: 1276416 | consumed tokens: 2614099968 | elapsed time per iteration (s): 5.43 | learning rate: 3.874E-04 | global batch size: 512 | lm loss: 3.219157E+00 | loss scale: 65536.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.312 | TFLOPs: 71.23 | +[default7]: iteration 2494/ 3814 | consumed samples: 1276928 | consumed tokens: 2615148544 | elapsed time per iteration (s): 5.42 | learning rate: 3.870E-04 | global batch size: 512 | lm loss: 3.191178E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.496 | TFLOPs: 71.37 | +[default7]: iteration 2495/ 3814 | consumed samples: 1277440 | consumed tokens: 2616197120 | elapsed time per iteration (s): 5.43 | learning rate: 3.866E-04 | global batch size: 512 | lm loss: 3.202292E+00 | loss scale: 65536.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.344 | TFLOPs: 71.26 | +[default7]: iteration 2496/ 3814 | consumed samples: 1277952 | consumed tokens: 2617245696 | elapsed time per iteration (s): 5.43 | learning rate: 3.863E-04 | global batch size: 512 | lm loss: 3.188599E+00 | loss scale: 65536.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.326 | TFLOPs: 71.25 | +[default7]: iteration 2497/ 3814 | consumed samples: 1278464 | consumed tokens: 2618294272 | elapsed time per iteration (s): 5.42 | learning rate: 3.859E-04 | global batch size: 512 | lm loss: 3.192162E+00 | loss scale: 65536.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 2498/ 3814 | consumed samples: 1278976 | consumed tokens: 2619342848 | elapsed time per iteration (s): 5.44 | learning rate: 3.855E-04 | global batch size: 512 | lm loss: 3.178178E+00 | loss scale: 65536.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.111 | TFLOPs: 71.08 | +[default7]: iteration 2499/ 3814 | consumed samples: 1279488 | consumed tokens: 2620391424 | elapsed time per iteration (s): 5.42 | learning rate: 3.851E-04 | global batch size: 512 | lm loss: 3.176007E+00 | loss scale: 65536.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.490 | TFLOPs: 71.37 | +[default0]:saving checkpoint at iteration 2500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 10:05:46,224] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2500 is begin to save! +[default0]:[2023-02-16 10:05:46,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,227] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_01-model_01-model_states.pt... +[default7]: iteration 2500/ 3814 | consumed samples: 1280000 | consumed tokens: 2621440000 | elapsed time per iteration (s): 5.41 | learning rate: 3.847E-04 | global batch size: 512 | lm loss: 3.188783E+00 | loss scale: 131072.0 | grad norm: 0.053 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default0]:[2023-02-16 10:05:46,379] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,380] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,374] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,434] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,435] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,429] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,430] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_05-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:46,485] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,485] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_06-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,490] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,490] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:46,543] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,544] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,537] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,599] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:46,653] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,654] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,594] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,595] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:46,646] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,646] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,710] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,711] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:46,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,766] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,700] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,701] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:46,756] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,757] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,821] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_12-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:46,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,875] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_13-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,812] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:46,866] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,867] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,928] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:46,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,919] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_14-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:46,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:46,974] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:46,984] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:46,984] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,038] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,025] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_16-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,084] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,085] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,097] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,098] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_17-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,160] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,268] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,268] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,201] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,255] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,256] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,330] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,331] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,318] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,371] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,371] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,396] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,396] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,459] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,460] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,439] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,524] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,504] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,504] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_24-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,564] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,565] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,586] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,643] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,643] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,630] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,631] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_26-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,690] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,691] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,703] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,704] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,757] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,758] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,748] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,748] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_28-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,813] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,814] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_29-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,868] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_30-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,801] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,802] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_29-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,858] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,859] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:47,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,924] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_31-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:47,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:47,982] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:47,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,914] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:47,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:47,971] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:48,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:48,037] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:48,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:48,026] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_33-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:48,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:48,079] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 10:05:48,090] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:48,090] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_34-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:48,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:48,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_35-model_00-model_states.pt... +[default1]:[2023-02-16 10:05:48,131] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:48,131] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:48,188] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:48,188] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 10:05:48,189] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 10:05:48,191] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_01_model_states.pt +[default1]:[2023-02-16 10:05:48,191] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 10:05:48,193] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_01_model_states.pt. +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:05:48,202] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:48,203] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 10:05:48,203] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 10:05:48,204] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_00_model_states.pt +[default0]:[2023-02-16 10:05:48,204] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 10:05:48,206] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:05:48,230] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:05:48,766] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,770] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,770] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,735] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,735] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,735] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,775] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,776] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,776] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,733] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,733] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,733] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,783] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,783] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,750] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,750] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,750] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,824] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,825] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,825] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,790] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,790] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,790] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,778] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,778] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,779] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,851] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,852] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,852] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,779] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,779] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,780] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,810] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,810] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,810] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,855] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,855] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,855] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,771] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,771] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,771] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,794] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,794] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,848] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,848] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,848] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,794] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,795] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,795] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,862] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,862] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,863] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,863] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,863] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,882] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,821] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,821] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,821] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,831] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,831] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,831] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,874] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,874] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,874] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,847] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,848] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,848] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,899] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,900] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,900] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,879] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,879] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,879] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,908] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,908] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,828] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,829] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,829] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,908] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,908] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,905] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,905] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,905] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,841] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,841] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,841] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,870] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,870] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,894] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,894] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,894] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,909] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,909] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,909] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,843] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,843] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,843] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,883] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,922] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,922] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,934] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,934] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,934] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,920] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,920] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,920] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,931] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,931] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,906] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,906] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,906] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,875] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,875] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,875] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,887] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,885] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,885] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,885] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,901] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,901] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,871] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,871] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,871] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default1]:[2023-02-16 10:05:48,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:05:48,925] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:05:48,925] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,908] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,909] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,909] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default6]:[2023-02-16 10:05:48,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:05:48,918] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:05:48,918] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:48,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:48,888] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:48,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,923] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,923] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,893] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,893] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,893] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,902] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,903] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,903] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:48,923] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:48,923] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:48,923] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default7]:[2023-02-16 10:05:48,950] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:05:48,950] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:05:48,950] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:48,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:48,924] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:48,925] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:48,929] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:48,929] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:48,929] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,964] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,965] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,965] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default3]:[2023-02-16 10:05:48,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:05:48,968] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:05:48,968] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default4]:[2023-02-16 10:05:49,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:05:49,006] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:05:49,006] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default2]:[2023-02-16 10:05:49,033] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:05:49,033] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:05:49,034] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]:[2023-02-16 10:05:49,102] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:05:49,102] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:05:49,102] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default5]:[2023-02-16 10:05:49,118] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:05:49,118] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2500/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:05:49,118] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2500 is ready now! +[default0]: successfully saved checkpoint at iteration 2500 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default7]:time (ms) | save-checkpoint: 2896.70 +[default7]: iteration 2501/ 3814 | consumed samples: 1280512 | consumed tokens: 2622488576 | elapsed time per iteration (s): 8.31 | learning rate: 3.843E-04 | global batch size: 512 | lm loss: 3.195160E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.605 | TFLOPs: 46.53 | +[default7]: iteration 2502/ 3814 | consumed samples: 1281024 | consumed tokens: 2623537152 | elapsed time per iteration (s): 5.41 | learning rate: 3.840E-04 | global batch size: 512 | lm loss: 3.192461E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 2503/ 3814 | consumed samples: 1281536 | consumed tokens: 2624585728 | elapsed time per iteration (s): 5.42 | learning rate: 3.836E-04 | global batch size: 512 | lm loss: 3.203226E+00 | loss scale: 131072.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.421 | TFLOPs: 71.32 | +[default7]: iteration 2504/ 3814 | consumed samples: 1282048 | consumed tokens: 2625634304 | elapsed time per iteration (s): 5.39 | learning rate: 3.832E-04 | global batch size: 512 | lm loss: 3.174150E+00 | loss scale: 131072.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.004 | TFLOPs: 71.76 | +[default7]: iteration 2505/ 3814 | consumed samples: 1282560 | consumed tokens: 2626682880 | elapsed time per iteration (s): 5.39 | learning rate: 3.828E-04 | global batch size: 512 | lm loss: 3.181704E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.948 | TFLOPs: 71.72 | +[default7]: iteration 2506/ 3814 | consumed samples: 1283072 | consumed tokens: 2627731456 | elapsed time per iteration (s): 5.39 | learning rate: 3.824E-04 | global batch size: 512 | lm loss: 3.200349E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.934 | TFLOPs: 71.70 | +[default7]: iteration 2507/ 3814 | consumed samples: 1283584 | consumed tokens: 2628780032 | elapsed time per iteration (s): 5.39 | learning rate: 3.821E-04 | global batch size: 512 | lm loss: 3.164663E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.043 | TFLOPs: 71.79 | +[default7]: iteration 2508/ 3814 | consumed samples: 1284096 | consumed tokens: 2629828608 | elapsed time per iteration (s): 5.40 | learning rate: 3.817E-04 | global batch size: 512 | lm loss: 3.201094E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.812 | TFLOPs: 71.61 | +[default7]: iteration 2509/ 3814 | consumed samples: 1284608 | consumed tokens: 2630877184 | elapsed time per iteration (s): 5.41 | learning rate: 3.813E-04 | global batch size: 512 | lm loss: 3.171432E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2510/ 3814 | consumed samples: 1285120 | consumed tokens: 2631925760 | elapsed time per iteration (s): 5.43 | learning rate: 3.809E-04 | global batch size: 512 | lm loss: 3.148268E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.343 | TFLOPs: 71.26 | +[default7]: iteration 2511/ 3814 | consumed samples: 1285632 | consumed tokens: 2632974336 | elapsed time per iteration (s): 5.41 | learning rate: 3.806E-04 | global batch size: 512 | lm loss: 3.178932E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.690 | TFLOPs: 71.52 | +[default7]: iteration 2512/ 3814 | consumed samples: 1286144 | consumed tokens: 2634022912 | elapsed time per iteration (s): 5.43 | learning rate: 3.802E-04 | global batch size: 512 | lm loss: 3.179472E+00 | loss scale: 131072.0 | grad norm: 0.097 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.286 | TFLOPs: 71.21 | +[default7]: iteration 2513/ 3814 | consumed samples: 1286656 | consumed tokens: 2635071488 | elapsed time per iteration (s): 5.42 | learning rate: 3.798E-04 | global batch size: 512 | lm loss: 3.198083E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.548 | TFLOPs: 71.41 | +[default7]: iteration 2514/ 3814 | consumed samples: 1287168 | consumed tokens: 2636120064 | elapsed time per iteration (s): 5.41 | learning rate: 3.794E-04 | global batch size: 512 | lm loss: 3.197573E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.667 | TFLOPs: 71.50 | +[default7]: iteration 2515/ 3814 | consumed samples: 1287680 | consumed tokens: 2637168640 | elapsed time per iteration (s): 5.42 | learning rate: 3.790E-04 | global batch size: 512 | lm loss: 3.195507E+00 | loss scale: 131072.0 | grad norm: 0.098 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.417 | TFLOPs: 71.31 | +[default7]: iteration 2516/ 3814 | consumed samples: 1288192 | consumed tokens: 2638217216 | elapsed time per iteration (s): 5.42 | learning rate: 3.787E-04 | global batch size: 512 | lm loss: 3.172777E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.492 | TFLOPs: 71.37 | +[default7]: iteration 2517/ 3814 | consumed samples: 1288704 | consumed tokens: 2639265792 | elapsed time per iteration (s): 5.43 | learning rate: 3.783E-04 | global batch size: 512 | lm loss: 3.171951E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.273 | TFLOPs: 71.21 | +[default7]: iteration 2518/ 3814 | consumed samples: 1289216 | consumed tokens: 2640314368 | elapsed time per iteration (s): 5.41 | learning rate: 3.779E-04 | global batch size: 512 | lm loss: 3.201187E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 2519/ 3814 | consumed samples: 1289728 | consumed tokens: 2641362944 | elapsed time per iteration (s): 5.43 | learning rate: 3.775E-04 | global batch size: 512 | lm loss: 3.172334E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.371 | TFLOPs: 71.28 | +[default7]: iteration 2520/ 3814 | consumed samples: 1290240 | consumed tokens: 2642411520 | elapsed time per iteration (s): 5.43 | learning rate: 3.771E-04 | global batch size: 512 | lm loss: 3.196500E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.286 | TFLOPs: 71.21 | +[default7]: iteration 2521/ 3814 | consumed samples: 1290752 | consumed tokens: 2643460096 | elapsed time per iteration (s): 5.41 | learning rate: 3.768E-04 | global batch size: 512 | lm loss: 3.188758E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2522/ 3814 | consumed samples: 1291264 | consumed tokens: 2644508672 | elapsed time per iteration (s): 5.42 | learning rate: 3.764E-04 | global batch size: 512 | lm loss: 3.187428E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.463 | TFLOPs: 71.35 | +[default7]: iteration 2523/ 3814 | consumed samples: 1291776 | consumed tokens: 2645557248 | elapsed time per iteration (s): 5.40 | learning rate: 3.760E-04 | global batch size: 512 | lm loss: 3.213989E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.861 | TFLOPs: 71.65 | +[default7]: iteration 2524/ 3814 | consumed samples: 1292288 | consumed tokens: 2646605824 | elapsed time per iteration (s): 5.39 | learning rate: 3.756E-04 | global batch size: 512 | lm loss: 3.210284E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.914 | TFLOPs: 71.69 | +[default7]: iteration 2525/ 3814 | consumed samples: 1292800 | consumed tokens: 2647654400 | elapsed time per iteration (s): 5.40 | learning rate: 3.753E-04 | global batch size: 512 | lm loss: 3.184805E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.866 | TFLOPs: 71.65 | +[default7]: iteration 2526/ 3814 | consumed samples: 1293312 | consumed tokens: 2648702976 | elapsed time per iteration (s): 5.39 | learning rate: 3.749E-04 | global batch size: 512 | lm loss: 3.182533E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.994 | TFLOPs: 71.75 | +[default7]: iteration 2527/ 3814 | consumed samples: 1293824 | consumed tokens: 2649751552 | elapsed time per iteration (s): 5.40 | learning rate: 3.745E-04 | global batch size: 512 | lm loss: 3.176567E+00 | loss scale: 131072.0 | grad norm: 0.125 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.807 | TFLOPs: 71.61 | +[default7]: iteration 2528/ 3814 | consumed samples: 1294336 | consumed tokens: 2650800128 | elapsed time per iteration (s): 5.43 | learning rate: 3.741E-04 | global batch size: 512 | lm loss: 3.189062E+00 | loss scale: 131072.0 | grad norm: 0.134 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.372 | TFLOPs: 71.28 | +[default7]: iteration 2529/ 3814 | consumed samples: 1294848 | consumed tokens: 2651848704 | elapsed time per iteration (s): 5.40 | learning rate: 3.738E-04 | global batch size: 512 | lm loss: 3.208876E+00 | loss scale: 131072.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.869 | TFLOPs: 71.66 | +[default7]: iteration 2530/ 3814 | consumed samples: 1295360 | consumed tokens: 2652897280 | elapsed time per iteration (s): 5.40 | learning rate: 3.734E-04 | global batch size: 512 | lm loss: 3.186252E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 2531/ 3814 | consumed samples: 1295872 | consumed tokens: 2653945856 | elapsed time per iteration (s): 5.40 | learning rate: 3.730E-04 | global batch size: 512 | lm loss: 3.182897E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.808 | TFLOPs: 71.61 | +[default7]: iteration 2532/ 3814 | consumed samples: 1296384 | consumed tokens: 2654994432 | elapsed time per iteration (s): 5.41 | learning rate: 3.726E-04 | global batch size: 512 | lm loss: 3.189166E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.666 | TFLOPs: 71.50 | +[default7]: iteration 2533/ 3814 | consumed samples: 1296896 | consumed tokens: 2656043008 | elapsed time per iteration (s): 5.40 | learning rate: 3.723E-04 | global batch size: 512 | lm loss: 3.178541E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.728 | TFLOPs: 71.55 | +[default7]: iteration 2534/ 3814 | consumed samples: 1297408 | consumed tokens: 2657091584 | elapsed time per iteration (s): 5.41 | learning rate: 3.719E-04 | global batch size: 512 | lm loss: 3.190828E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.587 | TFLOPs: 71.44 | +[default7]: iteration 2535/ 3814 | consumed samples: 1297920 | consumed tokens: 2658140160 | elapsed time per iteration (s): 5.41 | learning rate: 3.715E-04 | global batch size: 512 | lm loss: 3.208568E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.602 | TFLOPs: 71.45 | +[default7]: iteration 2536/ 3814 | consumed samples: 1298432 | consumed tokens: 2659188736 | elapsed time per iteration (s): 5.42 | learning rate: 3.711E-04 | global batch size: 512 | lm loss: 3.180841E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.494 | TFLOPs: 71.37 | +[default7]: iteration 2537/ 3814 | consumed samples: 1298944 | consumed tokens: 2660237312 | elapsed time per iteration (s): 5.43 | learning rate: 3.708E-04 | global batch size: 512 | lm loss: 3.165563E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 2538/ 3814 | consumed samples: 1299456 | consumed tokens: 2661285888 | elapsed time per iteration (s): 5.41 | learning rate: 3.704E-04 | global batch size: 512 | lm loss: 3.175280E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 2539/ 3814 | consumed samples: 1299968 | consumed tokens: 2662334464 | elapsed time per iteration (s): 5.43 | learning rate: 3.700E-04 | global batch size: 512 | lm loss: 3.172975E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.235 | TFLOPs: 71.18 | +[default7]: iteration 2540/ 3814 | consumed samples: 1300480 | consumed tokens: 2663383040 | elapsed time per iteration (s): 5.40 | learning rate: 3.696E-04 | global batch size: 512 | lm loss: 3.190147E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.851 | TFLOPs: 71.64 | +[default7]: iteration 2541/ 3814 | consumed samples: 1300992 | consumed tokens: 2664431616 | elapsed time per iteration (s): 5.40 | learning rate: 3.693E-04 | global batch size: 512 | lm loss: 3.157044E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.809 | TFLOPs: 71.61 | +[default7]: iteration 2542/ 3814 | consumed samples: 1301504 | consumed tokens: 2665480192 | elapsed time per iteration (s): 5.39 | learning rate: 3.689E-04 | global batch size: 512 | lm loss: 3.203219E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.949 | TFLOPs: 71.72 | +[default7]: iteration 2543/ 3814 | consumed samples: 1302016 | consumed tokens: 2666528768 | elapsed time per iteration (s): 5.41 | learning rate: 3.685E-04 | global batch size: 512 | lm loss: 3.176506E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.687 | TFLOPs: 71.52 | +[default7]: iteration 2544/ 3814 | consumed samples: 1302528 | consumed tokens: 2667577344 | elapsed time per iteration (s): 5.39 | learning rate: 3.681E-04 | global batch size: 512 | lm loss: 3.161597E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.025 | TFLOPs: 71.77 | +[default7]: iteration 2545/ 3814 | consumed samples: 1303040 | consumed tokens: 2668625920 | elapsed time per iteration (s): 5.40 | learning rate: 3.678E-04 | global batch size: 512 | lm loss: 3.209010E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.847 | TFLOPs: 71.64 | +[default7]: iteration 2546/ 3814 | consumed samples: 1303552 | consumed tokens: 2669674496 | elapsed time per iteration (s): 5.41 | learning rate: 3.674E-04 | global batch size: 512 | lm loss: 3.162033E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.647 | TFLOPs: 71.49 | +[default7]: iteration 2547/ 3814 | consumed samples: 1304064 | consumed tokens: 2670723072 | elapsed time per iteration (s): 5.40 | learning rate: 3.670E-04 | global batch size: 512 | lm loss: 3.195085E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.785 | TFLOPs: 71.59 | +[default7]: iteration 2548/ 3814 | consumed samples: 1304576 | consumed tokens: 2671771648 | elapsed time per iteration (s): 5.40 | learning rate: 3.666E-04 | global batch size: 512 | lm loss: 3.190204E+00 | loss scale: 131072.0 | grad norm: 0.157 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.755 | TFLOPs: 71.57 | +[default7]: iteration 2549/ 3814 | consumed samples: 1305088 | consumed tokens: 2672820224 | elapsed time per iteration (s): 5.40 | learning rate: 3.663E-04 | global batch size: 512 | lm loss: 3.168403E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.822 | TFLOPs: 71.62 | +[default7]: iteration 2550/ 3814 | consumed samples: 1305600 | consumed tokens: 2673868800 | elapsed time per iteration (s): 5.42 | learning rate: 3.659E-04 | global batch size: 512 | lm loss: 3.177892E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 2551/ 3814 | consumed samples: 1306112 | consumed tokens: 2674917376 | elapsed time per iteration (s): 5.41 | learning rate: 3.655E-04 | global batch size: 512 | lm loss: 3.160770E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.671 | TFLOPs: 71.51 | +[default7]: iteration 2552/ 3814 | consumed samples: 1306624 | consumed tokens: 2675965952 | elapsed time per iteration (s): 5.41 | learning rate: 3.651E-04 | global batch size: 512 | lm loss: 3.172326E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.590 | TFLOPs: 71.44 | +[default7]: iteration 2553/ 3814 | consumed samples: 1307136 | consumed tokens: 2677014528 | elapsed time per iteration (s): 5.39 | learning rate: 3.648E-04 | global batch size: 512 | lm loss: 3.181883E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.946 | TFLOPs: 71.71 | +[default7]: iteration 2554/ 3814 | consumed samples: 1307648 | consumed tokens: 2678063104 | elapsed time per iteration (s): 5.41 | learning rate: 3.644E-04 | global batch size: 512 | lm loss: 3.170623E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.617 | TFLOPs: 71.47 | +[default7]: iteration 2555/ 3814 | consumed samples: 1308160 | consumed tokens: 2679111680 | elapsed time per iteration (s): 5.43 | learning rate: 3.640E-04 | global batch size: 512 | lm loss: 3.169460E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 2556/ 3814 | consumed samples: 1308672 | consumed tokens: 2680160256 | elapsed time per iteration (s): 5.41 | learning rate: 3.637E-04 | global batch size: 512 | lm loss: 3.193744E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 2557/ 3814 | consumed samples: 1309184 | consumed tokens: 2681208832 | elapsed time per iteration (s): 5.40 | learning rate: 3.633E-04 | global batch size: 512 | lm loss: 3.172048E+00 | loss scale: 131072.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.800 | TFLOPs: 71.60 | +[default7]: iteration 2558/ 3814 | consumed samples: 1309696 | consumed tokens: 2682257408 | elapsed time per iteration (s): 5.40 | learning rate: 3.629E-04 | global batch size: 512 | lm loss: 3.171448E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.854 | TFLOPs: 71.64 | +[default7]: iteration 2559/ 3814 | consumed samples: 1310208 | consumed tokens: 2683305984 | elapsed time per iteration (s): 5.40 | learning rate: 3.625E-04 | global batch size: 512 | lm loss: 3.123714E+00 | loss scale: 131072.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.778 | TFLOPs: 71.59 | +[default7]: iteration 2560/ 3814 | consumed samples: 1310720 | consumed tokens: 2684354560 | elapsed time per iteration (s): 5.42 | learning rate: 3.622E-04 | global batch size: 512 | lm loss: 3.178096E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 2561/ 3814 | consumed samples: 1311232 | consumed tokens: 2685403136 | elapsed time per iteration (s): 5.42 | learning rate: 3.618E-04 | global batch size: 512 | lm loss: 3.193576E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.503 | TFLOPs: 71.38 | +[default7]: iteration 2562/ 3814 | consumed samples: 1311744 | consumed tokens: 2686451712 | elapsed time per iteration (s): 5.40 | learning rate: 3.614E-04 | global batch size: 512 | lm loss: 3.159118E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.817 | TFLOPs: 71.62 | +[default7]: iteration 2563/ 3814 | consumed samples: 1312256 | consumed tokens: 2687500288 | elapsed time per iteration (s): 5.40 | learning rate: 3.610E-04 | global batch size: 512 | lm loss: 3.157107E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.852 | TFLOPs: 71.64 | +[default7]: iteration 2564/ 3814 | consumed samples: 1312768 | consumed tokens: 2688548864 | elapsed time per iteration (s): 5.41 | learning rate: 3.607E-04 | global batch size: 512 | lm loss: 3.172955E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.700 | TFLOPs: 71.53 | +[default7]: iteration 2565/ 3814 | consumed samples: 1313280 | consumed tokens: 2689597440 | elapsed time per iteration (s): 5.41 | learning rate: 3.603E-04 | global batch size: 512 | lm loss: 3.167076E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.686 | TFLOPs: 71.52 | +[default7]: iteration 2566/ 3814 | consumed samples: 1313792 | consumed tokens: 2690646016 | elapsed time per iteration (s): 5.42 | learning rate: 3.599E-04 | global batch size: 512 | lm loss: 3.186246E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.513 | TFLOPs: 71.39 | +[default7]: iteration 2567/ 3814 | consumed samples: 1314304 | consumed tokens: 2691694592 | elapsed time per iteration (s): 5.41 | learning rate: 3.596E-04 | global batch size: 512 | lm loss: 3.182973E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.712 | TFLOPs: 71.54 | +[default7]: iteration 2568/ 3814 | consumed samples: 1314816 | consumed tokens: 2692743168 | elapsed time per iteration (s): 5.40 | learning rate: 3.592E-04 | global batch size: 512 | lm loss: 3.160957E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.867 | TFLOPs: 71.65 | +[default7]: iteration 2569/ 3814 | consumed samples: 1315328 | consumed tokens: 2693791744 | elapsed time per iteration (s): 5.41 | learning rate: 3.588E-04 | global batch size: 512 | lm loss: 3.179964E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.653 | TFLOPs: 71.49 | +[default7]: iteration 2570/ 3814 | consumed samples: 1315840 | consumed tokens: 2694840320 | elapsed time per iteration (s): 5.40 | learning rate: 3.585E-04 | global batch size: 512 | lm loss: 3.185266E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.877 | TFLOPs: 71.66 | +[default7]: iteration 2571/ 3814 | consumed samples: 1316352 | consumed tokens: 2695888896 | elapsed time per iteration (s): 5.41 | learning rate: 3.581E-04 | global batch size: 512 | lm loss: 3.174951E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.581 | TFLOPs: 71.44 | +[default7]: iteration 2572/ 3814 | consumed samples: 1316864 | consumed tokens: 2696937472 | elapsed time per iteration (s): 5.40 | learning rate: 3.577E-04 | global batch size: 512 | lm loss: 3.181678E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.875 | TFLOPs: 71.66 | +[default7]: iteration 2573/ 3814 | consumed samples: 1317376 | consumed tokens: 2697986048 | elapsed time per iteration (s): 5.42 | learning rate: 3.573E-04 | global batch size: 512 | lm loss: 3.189087E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 2574/ 3814 | consumed samples: 1317888 | consumed tokens: 2699034624 | elapsed time per iteration (s): 5.39 | learning rate: 3.570E-04 | global batch size: 512 | lm loss: 3.192207E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.952 | TFLOPs: 71.72 | +[default7]: iteration 2575/ 3814 | consumed samples: 1318400 | consumed tokens: 2700083200 | elapsed time per iteration (s): 5.40 | learning rate: 3.566E-04 | global batch size: 512 | lm loss: 3.152514E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 2576/ 3814 | consumed samples: 1318912 | consumed tokens: 2701131776 | elapsed time per iteration (s): 5.43 | learning rate: 3.562E-04 | global batch size: 512 | lm loss: 3.175940E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.325 | TFLOPs: 71.24 | +[default7]: iteration 2577/ 3814 | consumed samples: 1319424 | consumed tokens: 2702180352 | elapsed time per iteration (s): 5.42 | learning rate: 3.559E-04 | global batch size: 512 | lm loss: 3.184307E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.482 | TFLOPs: 71.36 | +[default7]: iteration 2578/ 3814 | consumed samples: 1319936 | consumed tokens: 2703228928 | elapsed time per iteration (s): 5.41 | learning rate: 3.555E-04 | global batch size: 512 | lm loss: 3.216749E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 2579/ 3814 | consumed samples: 1320448 | consumed tokens: 2704277504 | elapsed time per iteration (s): 5.42 | learning rate: 3.551E-04 | global batch size: 512 | lm loss: 3.196246E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.529 | TFLOPs: 71.40 | +[default7]: iteration 2580/ 3814 | consumed samples: 1320960 | consumed tokens: 2705326080 | elapsed time per iteration (s): 5.42 | learning rate: 3.548E-04 | global batch size: 512 | lm loss: 3.158713E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.414 | TFLOPs: 71.31 | +[default7]: iteration 2581/ 3814 | consumed samples: 1321472 | consumed tokens: 2706374656 | elapsed time per iteration (s): 5.40 | learning rate: 3.544E-04 | global batch size: 512 | lm loss: 3.143364E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.857 | TFLOPs: 71.65 | +[default7]: iteration 2582/ 3814 | consumed samples: 1321984 | consumed tokens: 2707423232 | elapsed time per iteration (s): 5.41 | learning rate: 3.540E-04 | global batch size: 512 | lm loss: 3.191382E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.591 | TFLOPs: 71.45 | +[default7]: iteration 2583/ 3814 | consumed samples: 1322496 | consumed tokens: 2708471808 | elapsed time per iteration (s): 5.41 | learning rate: 3.537E-04 | global batch size: 512 | lm loss: 3.167004E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.635 | TFLOPs: 71.48 | +[default7]: iteration 2584/ 3814 | consumed samples: 1323008 | consumed tokens: 2709520384 | elapsed time per iteration (s): 5.42 | learning rate: 3.533E-04 | global batch size: 512 | lm loss: 3.161398E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.543 | TFLOPs: 71.41 | +[default7]: iteration 2585/ 3814 | consumed samples: 1323520 | consumed tokens: 2710568960 | elapsed time per iteration (s): 5.41 | learning rate: 3.529E-04 | global batch size: 512 | lm loss: 3.161427E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.617 | TFLOPs: 71.47 | +[default7]: iteration 2586/ 3814 | consumed samples: 1324032 | consumed tokens: 2711617536 | elapsed time per iteration (s): 5.42 | learning rate: 3.526E-04 | global batch size: 512 | lm loss: 3.157619E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.405 | TFLOPs: 71.31 | +[default7]: iteration 2587/ 3814 | consumed samples: 1324544 | consumed tokens: 2712666112 | elapsed time per iteration (s): 5.44 | learning rate: 3.522E-04 | global batch size: 512 | lm loss: 3.164800E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.101 | TFLOPs: 71.08 | +[default7]: iteration 2588/ 3814 | consumed samples: 1325056 | consumed tokens: 2713714688 | elapsed time per iteration (s): 5.46 | learning rate: 3.518E-04 | global batch size: 512 | lm loss: 3.159807E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.708 | TFLOPs: 70.78 | +[default7]: iteration 2589/ 3814 | consumed samples: 1325568 | consumed tokens: 2714763264 | elapsed time per iteration (s): 5.42 | learning rate: 3.515E-04 | global batch size: 512 | lm loss: 3.149806E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default7]: iteration 2590/ 3814 | consumed samples: 1326080 | consumed tokens: 2715811840 | elapsed time per iteration (s): 5.40 | learning rate: 3.511E-04 | global batch size: 512 | lm loss: 3.167151E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.792 | TFLOPs: 71.60 | +[default7]: iteration 2591/ 3814 | consumed samples: 1326592 | consumed tokens: 2716860416 | elapsed time per iteration (s): 5.41 | learning rate: 3.507E-04 | global batch size: 512 | lm loss: 3.168749E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.569 | TFLOPs: 71.43 | +[default7]: iteration 2592/ 3814 | consumed samples: 1327104 | consumed tokens: 2717908992 | elapsed time per iteration (s): 5.41 | learning rate: 3.504E-04 | global batch size: 512 | lm loss: 3.185427E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.656 | TFLOPs: 71.49 | +[default7]: iteration 2593/ 3814 | consumed samples: 1327616 | consumed tokens: 2718957568 | elapsed time per iteration (s): 5.39 | learning rate: 3.500E-04 | global batch size: 512 | lm loss: 3.158608E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.071 | TFLOPs: 71.81 | +[default7]: iteration 2594/ 3814 | consumed samples: 1328128 | consumed tokens: 2720006144 | elapsed time per iteration (s): 5.40 | learning rate: 3.496E-04 | global batch size: 512 | lm loss: 3.181802E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.882 | TFLOPs: 71.67 | +[default7]: iteration 2595/ 3814 | consumed samples: 1328640 | consumed tokens: 2721054720 | elapsed time per iteration (s): 5.40 | learning rate: 3.493E-04 | global batch size: 512 | lm loss: 3.174217E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.730 | TFLOPs: 71.55 | +[default7]: iteration 2596/ 3814 | consumed samples: 1329152 | consumed tokens: 2722103296 | elapsed time per iteration (s): 5.40 | learning rate: 3.489E-04 | global batch size: 512 | lm loss: 3.154041E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.775 | TFLOPs: 71.58 | +[default7]: iteration 2597/ 3814 | consumed samples: 1329664 | consumed tokens: 2723151872 | elapsed time per iteration (s): 5.40 | learning rate: 3.485E-04 | global batch size: 512 | lm loss: 3.185532E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.739 | TFLOPs: 71.56 | +[default7]: iteration 2598/ 3814 | consumed samples: 1330176 | consumed tokens: 2724200448 | elapsed time per iteration (s): 5.41 | learning rate: 3.482E-04 | global batch size: 512 | lm loss: 3.167058E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 2599/ 3814 | consumed samples: 1330688 | consumed tokens: 2725249024 | elapsed time per iteration (s): 5.41 | learning rate: 3.478E-04 | global batch size: 512 | lm loss: 3.173065E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default7]: iteration 2600/ 3814 | consumed samples: 1331200 | consumed tokens: 2726297600 | elapsed time per iteration (s): 5.41 | learning rate: 3.474E-04 | global batch size: 512 | lm loss: 3.136528E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 2601/ 3814 | consumed samples: 1331712 | consumed tokens: 2727346176 | elapsed time per iteration (s): 5.42 | learning rate: 3.471E-04 | global batch size: 512 | lm loss: 3.168988E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.414 | TFLOPs: 71.31 | +[default7]: iteration 2602/ 3814 | consumed samples: 1332224 | consumed tokens: 2728394752 | elapsed time per iteration (s): 5.39 | learning rate: 3.467E-04 | global batch size: 512 | lm loss: 3.160792E+00 | loss scale: 131072.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.030 | TFLOPs: 71.78 | +[default7]: iteration 2603/ 3814 | consumed samples: 1332736 | consumed tokens: 2729443328 | elapsed time per iteration (s): 5.39 | learning rate: 3.463E-04 | global batch size: 512 | lm loss: 3.179843E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.909 | TFLOPs: 71.69 | +[default7]: iteration 2604/ 3814 | consumed samples: 1333248 | consumed tokens: 2730491904 | elapsed time per iteration (s): 5.42 | learning rate: 3.460E-04 | global batch size: 512 | lm loss: 3.159894E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.422 | TFLOPs: 71.32 | +[default7]: iteration 2605/ 3814 | consumed samples: 1333760 | consumed tokens: 2731540480 | elapsed time per iteration (s): 5.42 | learning rate: 3.456E-04 | global batch size: 512 | lm loss: 3.137770E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 2606/ 3814 | consumed samples: 1334272 | consumed tokens: 2732589056 | elapsed time per iteration (s): 5.42 | learning rate: 3.452E-04 | global batch size: 512 | lm loss: 3.158632E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.522 | TFLOPs: 71.39 | +[default7]: iteration 2607/ 3814 | consumed samples: 1334784 | consumed tokens: 2733637632 | elapsed time per iteration (s): 5.42 | learning rate: 3.449E-04 | global batch size: 512 | lm loss: 3.195722E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.484 | TFLOPs: 71.36 | +[default7]: iteration 2608/ 3814 | consumed samples: 1335296 | consumed tokens: 2734686208 | elapsed time per iteration (s): 5.39 | learning rate: 3.445E-04 | global batch size: 512 | lm loss: 3.128863E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.977 | TFLOPs: 71.74 | +[default7]: iteration 2609/ 3814 | consumed samples: 1335808 | consumed tokens: 2735734784 | elapsed time per iteration (s): 5.42 | learning rate: 3.441E-04 | global batch size: 512 | lm loss: 3.184855E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.444 | TFLOPs: 71.33 | +[default7]: iteration 2610/ 3814 | consumed samples: 1336320 | consumed tokens: 2736783360 | elapsed time per iteration (s): 5.40 | learning rate: 3.438E-04 | global batch size: 512 | lm loss: 3.157170E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.832 | TFLOPs: 71.63 | +[default7]: iteration 2611/ 3814 | consumed samples: 1336832 | consumed tokens: 2737831936 | elapsed time per iteration (s): 5.40 | learning rate: 3.434E-04 | global batch size: 512 | lm loss: 3.171223E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 2612/ 3814 | consumed samples: 1337344 | consumed tokens: 2738880512 | elapsed time per iteration (s): 5.42 | learning rate: 3.431E-04 | global batch size: 512 | lm loss: 3.181678E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.382 | TFLOPs: 71.29 | +[default7]: iteration 2613/ 3814 | consumed samples: 1337856 | consumed tokens: 2739929088 | elapsed time per iteration (s): 5.41 | learning rate: 3.427E-04 | global batch size: 512 | lm loss: 3.143527E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.589 | TFLOPs: 71.44 | +[default7]: iteration 2614/ 3814 | consumed samples: 1338368 | consumed tokens: 2740977664 | elapsed time per iteration (s): 5.41 | learning rate: 3.423E-04 | global batch size: 512 | lm loss: 3.158374E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.648 | TFLOPs: 71.49 | +[default7]: iteration 2615/ 3814 | consumed samples: 1338880 | consumed tokens: 2742026240 | elapsed time per iteration (s): 5.41 | learning rate: 3.420E-04 | global batch size: 512 | lm loss: 3.187526E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 2616/ 3814 | consumed samples: 1339392 | consumed tokens: 2743074816 | elapsed time per iteration (s): 5.40 | learning rate: 3.416E-04 | global batch size: 512 | lm loss: 3.161347E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.850 | TFLOPs: 71.64 | +[default7]: iteration 2617/ 3814 | consumed samples: 1339904 | consumed tokens: 2744123392 | elapsed time per iteration (s): 5.40 | learning rate: 3.412E-04 | global batch size: 512 | lm loss: 3.181497E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.805 | TFLOPs: 71.61 | +[default7]: iteration 2618/ 3814 | consumed samples: 1340416 | consumed tokens: 2745171968 | elapsed time per iteration (s): 5.41 | learning rate: 3.409E-04 | global batch size: 512 | lm loss: 3.173143E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.698 | TFLOPs: 71.53 | +[default7]: iteration 2619/ 3814 | consumed samples: 1340928 | consumed tokens: 2746220544 | elapsed time per iteration (s): 5.43 | learning rate: 3.405E-04 | global batch size: 512 | lm loss: 3.156087E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.371 | TFLOPs: 71.28 | +[default7]: iteration 2620/ 3814 | consumed samples: 1341440 | consumed tokens: 2747269120 | elapsed time per iteration (s): 5.43 | learning rate: 3.402E-04 | global batch size: 512 | lm loss: 3.138908E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.283 | TFLOPs: 71.21 | +[default7]: iteration 2621/ 3814 | consumed samples: 1341952 | consumed tokens: 2748317696 | elapsed time per iteration (s): 5.40 | learning rate: 3.398E-04 | global batch size: 512 | lm loss: 3.161282E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.763 | TFLOPs: 71.58 | +[default7]: iteration 2622/ 3814 | consumed samples: 1342464 | consumed tokens: 2749366272 | elapsed time per iteration (s): 5.41 | learning rate: 3.394E-04 | global batch size: 512 | lm loss: 3.129292E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.675 | TFLOPs: 71.51 | +[default7]: iteration 2623/ 3814 | consumed samples: 1342976 | consumed tokens: 2750414848 | elapsed time per iteration (s): 5.42 | learning rate: 3.391E-04 | global batch size: 512 | lm loss: 3.169744E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.448 | TFLOPs: 71.34 | +[default7]: iteration 2624/ 3814 | consumed samples: 1343488 | consumed tokens: 2751463424 | elapsed time per iteration (s): 5.41 | learning rate: 3.387E-04 | global batch size: 512 | lm loss: 3.176013E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 2625/ 3814 | consumed samples: 1344000 | consumed tokens: 2752512000 | elapsed time per iteration (s): 5.41 | learning rate: 3.384E-04 | global batch size: 512 | lm loss: 3.155619E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.562 | TFLOPs: 71.42 | +[default7]: iteration 2626/ 3814 | consumed samples: 1344512 | consumed tokens: 2753560576 | elapsed time per iteration (s): 5.41 | learning rate: 3.380E-04 | global batch size: 512 | lm loss: 3.169105E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 2627/ 3814 | consumed samples: 1345024 | consumed tokens: 2754609152 | elapsed time per iteration (s): 5.41 | learning rate: 3.376E-04 | global batch size: 512 | lm loss: 3.153517E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2628/ 3814 | consumed samples: 1345536 | consumed tokens: 2755657728 | elapsed time per iteration (s): 5.42 | learning rate: 3.373E-04 | global batch size: 512 | lm loss: 3.174085E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.441 | TFLOPs: 71.33 | +[default7]: iteration 2629/ 3814 | consumed samples: 1346048 | consumed tokens: 2756706304 | elapsed time per iteration (s): 5.41 | learning rate: 3.369E-04 | global batch size: 512 | lm loss: 3.148783E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2630/ 3814 | consumed samples: 1346560 | consumed tokens: 2757754880 | elapsed time per iteration (s): 5.42 | learning rate: 3.365E-04 | global batch size: 512 | lm loss: 3.173954E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.526 | TFLOPs: 71.40 | +[default7]: iteration 2631/ 3814 | consumed samples: 1347072 | consumed tokens: 2758803456 | elapsed time per iteration (s): 5.43 | learning rate: 3.362E-04 | global batch size: 512 | lm loss: 3.169420E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.279 | TFLOPs: 71.21 | +[default7]: iteration 2632/ 3814 | consumed samples: 1347584 | consumed tokens: 2759852032 | elapsed time per iteration (s): 5.42 | learning rate: 3.358E-04 | global batch size: 512 | lm loss: 3.190199E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.473 | TFLOPs: 71.36 | +[default7]: iteration 2633/ 3814 | consumed samples: 1348096 | consumed tokens: 2760900608 | elapsed time per iteration (s): 5.41 | learning rate: 3.355E-04 | global batch size: 512 | lm loss: 3.159721E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.652 | TFLOPs: 71.49 | +[default7]: iteration 2634/ 3814 | consumed samples: 1348608 | consumed tokens: 2761949184 | elapsed time per iteration (s): 5.41 | learning rate: 3.351E-04 | global batch size: 512 | lm loss: 3.147661E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.616 | TFLOPs: 71.46 | +[default7]: iteration 2635/ 3814 | consumed samples: 1349120 | consumed tokens: 2762997760 | elapsed time per iteration (s): 5.41 | learning rate: 3.348E-04 | global batch size: 512 | lm loss: 3.180851E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.634 | TFLOPs: 71.48 | +[default7]: iteration 2636/ 3814 | consumed samples: 1349632 | consumed tokens: 2764046336 | elapsed time per iteration (s): 5.42 | learning rate: 3.344E-04 | global batch size: 512 | lm loss: 3.134187E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.524 | TFLOPs: 71.40 | +[default7]: iteration 2637/ 3814 | consumed samples: 1350144 | consumed tokens: 2765094912 | elapsed time per iteration (s): 5.43 | learning rate: 3.340E-04 | global batch size: 512 | lm loss: 3.162183E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.371 | TFLOPs: 71.28 | +[default7]: iteration 2638/ 3814 | consumed samples: 1350656 | consumed tokens: 2766143488 | elapsed time per iteration (s): 5.42 | learning rate: 3.337E-04 | global batch size: 512 | lm loss: 3.149117E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.401 | TFLOPs: 71.30 | +[default7]: iteration 2639/ 3814 | consumed samples: 1351168 | consumed tokens: 2767192064 | elapsed time per iteration (s): 5.42 | learning rate: 3.333E-04 | global batch size: 512 | lm loss: 3.157156E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 2640/ 3814 | consumed samples: 1351680 | consumed tokens: 2768240640 | elapsed time per iteration (s): 5.42 | learning rate: 3.330E-04 | global batch size: 512 | lm loss: 3.159313E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.396 | TFLOPs: 71.30 | +[default7]: iteration 2641/ 3814 | consumed samples: 1352192 | consumed tokens: 2769289216 | elapsed time per iteration (s): 5.41 | learning rate: 3.326E-04 | global batch size: 512 | lm loss: 3.135994E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.667 | TFLOPs: 71.50 | +[default7]: iteration 2642/ 3814 | consumed samples: 1352704 | consumed tokens: 2770337792 | elapsed time per iteration (s): 5.41 | learning rate: 3.322E-04 | global batch size: 512 | lm loss: 3.163043E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.698 | TFLOPs: 71.53 | +[default7]: iteration 2643/ 3814 | consumed samples: 1353216 | consumed tokens: 2771386368 | elapsed time per iteration (s): 5.41 | learning rate: 3.319E-04 | global batch size: 512 | lm loss: 3.162616E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 2644/ 3814 | consumed samples: 1353728 | consumed tokens: 2772434944 | elapsed time per iteration (s): 5.41 | learning rate: 3.315E-04 | global batch size: 512 | lm loss: 3.131086E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.721 | TFLOPs: 71.54 | +[default7]: iteration 2645/ 3814 | consumed samples: 1354240 | consumed tokens: 2773483520 | elapsed time per iteration (s): 5.42 | learning rate: 3.312E-04 | global batch size: 512 | lm loss: 3.151049E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.407 | TFLOPs: 71.31 | +[default7]: iteration 2646/ 3814 | consumed samples: 1354752 | consumed tokens: 2774532096 | elapsed time per iteration (s): 5.42 | learning rate: 3.308E-04 | global batch size: 512 | lm loss: 3.141983E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.458 | TFLOPs: 71.35 | +[default7]: iteration 2647/ 3814 | consumed samples: 1355264 | consumed tokens: 2775580672 | elapsed time per iteration (s): 5.42 | learning rate: 3.305E-04 | global batch size: 512 | lm loss: 3.162658E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.452 | TFLOPs: 71.34 | +[default7]: iteration 2648/ 3814 | consumed samples: 1355776 | consumed tokens: 2776629248 | elapsed time per iteration (s): 5.44 | learning rate: 3.301E-04 | global batch size: 512 | lm loss: 3.139003E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.090 | TFLOPs: 71.07 | +[default7]: iteration 2649/ 3814 | consumed samples: 1356288 | consumed tokens: 2777677824 | elapsed time per iteration (s): 5.43 | learning rate: 3.297E-04 | global batch size: 512 | lm loss: 3.152604E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.329 | TFLOPs: 71.25 | +[default7]: iteration 2650/ 3814 | consumed samples: 1356800 | consumed tokens: 2778726400 | elapsed time per iteration (s): 5.43 | learning rate: 3.294E-04 | global batch size: 512 | lm loss: 3.147076E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.362 | TFLOPs: 71.27 | +[default7]: iteration 2651/ 3814 | consumed samples: 1357312 | consumed tokens: 2779774976 | elapsed time per iteration (s): 5.41 | learning rate: 3.290E-04 | global batch size: 512 | lm loss: 3.128495E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.570 | TFLOPs: 71.43 | +[default7]: iteration 2652/ 3814 | consumed samples: 1357824 | consumed tokens: 2780823552 | elapsed time per iteration (s): 5.41 | learning rate: 3.287E-04 | global batch size: 512 | lm loss: 3.145342E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 2653/ 3814 | consumed samples: 1358336 | consumed tokens: 2781872128 | elapsed time per iteration (s): 5.42 | learning rate: 3.283E-04 | global batch size: 512 | lm loss: 3.167018E+00 | loss scale: 131072.0 | grad norm: 0.101 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.431 | TFLOPs: 71.32 | +[default7]: iteration 2654/ 3814 | consumed samples: 1358848 | consumed tokens: 2782920704 | elapsed time per iteration (s): 5.42 | learning rate: 3.280E-04 | global batch size: 512 | lm loss: 3.171985E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.468 | TFLOPs: 71.35 | +[default7]: iteration 2655/ 3814 | consumed samples: 1359360 | consumed tokens: 2783969280 | elapsed time per iteration (s): 5.41 | learning rate: 3.276E-04 | global batch size: 512 | lm loss: 3.164065E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 2656/ 3814 | consumed samples: 1359872 | consumed tokens: 2785017856 | elapsed time per iteration (s): 5.41 | learning rate: 3.272E-04 | global batch size: 512 | lm loss: 3.167065E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2657/ 3814 | consumed samples: 1360384 | consumed tokens: 2786066432 | elapsed time per iteration (s): 5.42 | learning rate: 3.269E-04 | global batch size: 512 | lm loss: 3.173287E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 2658/ 3814 | consumed samples: 1360896 | consumed tokens: 2787115008 | elapsed time per iteration (s): 5.43 | learning rate: 3.265E-04 | global batch size: 512 | lm loss: 3.166973E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.208 | TFLOPs: 71.16 | +[default7]: iteration 2659/ 3814 | consumed samples: 1361408 | consumed tokens: 2788163584 | elapsed time per iteration (s): 5.43 | learning rate: 3.262E-04 | global batch size: 512 | lm loss: 3.164453E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.369 | TFLOPs: 71.28 | +[default7]: iteration 2660/ 3814 | consumed samples: 1361920 | consumed tokens: 2789212160 | elapsed time per iteration (s): 5.41 | learning rate: 3.258E-04 | global batch size: 512 | lm loss: 3.158562E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.555 | TFLOPs: 71.42 | +[default7]: iteration 2661/ 3814 | consumed samples: 1362432 | consumed tokens: 2790260736 | elapsed time per iteration (s): 5.40 | learning rate: 3.255E-04 | global batch size: 512 | lm loss: 3.176310E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.827 | TFLOPs: 71.62 | +[default7]: iteration 2662/ 3814 | consumed samples: 1362944 | consumed tokens: 2791309312 | elapsed time per iteration (s): 5.40 | learning rate: 3.251E-04 | global batch size: 512 | lm loss: 3.154098E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.733 | TFLOPs: 71.55 | +[default7]: iteration 2663/ 3814 | consumed samples: 1363456 | consumed tokens: 2792357888 | elapsed time per iteration (s): 5.41 | learning rate: 3.248E-04 | global batch size: 512 | lm loss: 3.134015E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.593 | TFLOPs: 71.45 | +[default7]: iteration 2664/ 3814 | consumed samples: 1363968 | consumed tokens: 2793406464 | elapsed time per iteration (s): 5.38 | learning rate: 3.244E-04 | global batch size: 512 | lm loss: 3.120403E+00 | loss scale: 131072.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.169 | TFLOPs: 71.88 | +[default7]: iteration 2665/ 3814 | consumed samples: 1364480 | consumed tokens: 2794455040 | elapsed time per iteration (s): 5.41 | learning rate: 3.241E-04 | global batch size: 512 | lm loss: 3.128974E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.580 | TFLOPs: 71.44 | +[default7]: iteration 2666/ 3814 | consumed samples: 1364992 | consumed tokens: 2795503616 | elapsed time per iteration (s): 5.41 | learning rate: 3.237E-04 | global batch size: 512 | lm loss: 3.144550E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.715 | TFLOPs: 71.54 | +[default7]: iteration 2667/ 3814 | consumed samples: 1365504 | consumed tokens: 2796552192 | elapsed time per iteration (s): 5.40 | learning rate: 3.234E-04 | global batch size: 512 | lm loss: 3.145542E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.727 | TFLOPs: 71.55 | +[default7]: iteration 2668/ 3814 | consumed samples: 1366016 | consumed tokens: 2797600768 | elapsed time per iteration (s): 5.41 | learning rate: 3.230E-04 | global batch size: 512 | lm loss: 3.173755E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.560 | TFLOPs: 71.42 | +[default7]: iteration 2669/ 3814 | consumed samples: 1366528 | consumed tokens: 2798649344 | elapsed time per iteration (s): 5.42 | learning rate: 3.226E-04 | global batch size: 512 | lm loss: 3.158331E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.446 | TFLOPs: 71.34 | +[default7]: iteration 2670/ 3814 | consumed samples: 1367040 | consumed tokens: 2799697920 | elapsed time per iteration (s): 5.41 | learning rate: 3.223E-04 | global batch size: 512 | lm loss: 3.140848E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.564 | TFLOPs: 71.43 | +[default7]: iteration 2671/ 3814 | consumed samples: 1367552 | consumed tokens: 2800746496 | elapsed time per iteration (s): 5.43 | learning rate: 3.219E-04 | global batch size: 512 | lm loss: 3.153083E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.337 | TFLOPs: 71.25 | +[default7]: iteration 2672/ 3814 | consumed samples: 1368064 | consumed tokens: 2801795072 | elapsed time per iteration (s): 5.42 | learning rate: 3.216E-04 | global batch size: 512 | lm loss: 3.151720E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.385 | TFLOPs: 71.29 | +[default7]: iteration 2673/ 3814 | consumed samples: 1368576 | consumed tokens: 2802843648 | elapsed time per iteration (s): 5.42 | learning rate: 3.212E-04 | global batch size: 512 | lm loss: 3.159705E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.551 | TFLOPs: 71.42 | +[default7]: iteration 2674/ 3814 | consumed samples: 1369088 | consumed tokens: 2803892224 | elapsed time per iteration (s): 5.41 | learning rate: 3.209E-04 | global batch size: 512 | lm loss: 3.140918E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.715 | TFLOPs: 71.54 | +[default7]: iteration 2675/ 3814 | consumed samples: 1369600 | consumed tokens: 2804940800 | elapsed time per iteration (s): 5.40 | learning rate: 3.205E-04 | global batch size: 512 | lm loss: 3.127526E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.846 | TFLOPs: 71.64 | +[default7]: iteration 2676/ 3814 | consumed samples: 1370112 | consumed tokens: 2805989376 | elapsed time per iteration (s): 5.42 | learning rate: 3.202E-04 | global batch size: 512 | lm loss: 3.149450E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.547 | TFLOPs: 71.41 | +[default7]: iteration 2677/ 3814 | consumed samples: 1370624 | consumed tokens: 2807037952 | elapsed time per iteration (s): 5.42 | learning rate: 3.198E-04 | global batch size: 512 | lm loss: 3.185223E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.531 | TFLOPs: 71.40 | +[default7]: iteration 2678/ 3814 | consumed samples: 1371136 | consumed tokens: 2808086528 | elapsed time per iteration (s): 5.42 | learning rate: 3.195E-04 | global batch size: 512 | lm loss: 3.193391E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.509 | TFLOPs: 71.38 | +[default7]: iteration 2679/ 3814 | consumed samples: 1371648 | consumed tokens: 2809135104 | elapsed time per iteration (s): 5.44 | learning rate: 3.191E-04 | global batch size: 512 | lm loss: 3.150424E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.172 | TFLOPs: 71.13 | +[default7]: iteration 2680/ 3814 | consumed samples: 1372160 | consumed tokens: 2810183680 | elapsed time per iteration (s): 5.42 | learning rate: 3.188E-04 | global batch size: 512 | lm loss: 3.159990E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.424 | TFLOPs: 71.32 | +[default7]: iteration 2681/ 3814 | consumed samples: 1372672 | consumed tokens: 2811232256 | elapsed time per iteration (s): 5.40 | learning rate: 3.184E-04 | global batch size: 512 | lm loss: 3.108489E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.795 | TFLOPs: 71.60 | +[default7]: iteration 2682/ 3814 | consumed samples: 1373184 | consumed tokens: 2812280832 | elapsed time per iteration (s): 5.40 | learning rate: 3.181E-04 | global batch size: 512 | lm loss: 3.157873E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.738 | TFLOPs: 71.56 | +[default7]: iteration 2683/ 3814 | consumed samples: 1373696 | consumed tokens: 2813329408 | elapsed time per iteration (s): 5.40 | learning rate: 3.177E-04 | global batch size: 512 | lm loss: 3.129544E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.762 | TFLOPs: 71.57 | +[default7]: iteration 2684/ 3814 | consumed samples: 1374208 | consumed tokens: 2814377984 | elapsed time per iteration (s): 5.44 | learning rate: 3.174E-04 | global batch size: 512 | lm loss: 3.121325E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.170 | TFLOPs: 71.13 | +[default7]: iteration 2685/ 3814 | consumed samples: 1374720 | consumed tokens: 2815426560 | elapsed time per iteration (s): 5.40 | learning rate: 3.170E-04 | global batch size: 512 | lm loss: 3.139589E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.808 | TFLOPs: 71.61 | +[default7]: iteration 2686/ 3814 | consumed samples: 1375232 | consumed tokens: 2816475136 | elapsed time per iteration (s): 5.43 | learning rate: 3.167E-04 | global batch size: 512 | lm loss: 3.146818E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.284 | TFLOPs: 71.21 | +[default7]: iteration 2687/ 3814 | consumed samples: 1375744 | consumed tokens: 2817523712 | elapsed time per iteration (s): 5.39 | learning rate: 3.163E-04 | global batch size: 512 | lm loss: 3.139902E+00 | loss scale: 131072.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.939 | TFLOPs: 71.71 | +[default7]: iteration 2688/ 3814 | consumed samples: 1376256 | consumed tokens: 2818572288 | elapsed time per iteration (s): 5.39 | learning rate: 3.160E-04 | global batch size: 512 | lm loss: 3.164608E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.030 | TFLOPs: 71.78 | +[default7]: iteration 2689/ 3814 | consumed samples: 1376768 | consumed tokens: 2819620864 | elapsed time per iteration (s): 5.41 | learning rate: 3.156E-04 | global batch size: 512 | lm loss: 3.158787E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.575 | TFLOPs: 71.43 | +[default7]: iteration 2690/ 3814 | consumed samples: 1377280 | consumed tokens: 2820669440 | elapsed time per iteration (s): 5.45 | learning rate: 3.153E-04 | global batch size: 512 | lm loss: 3.129646E+00 | loss scale: 131072.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.013 | TFLOPs: 71.01 | +[default7]: iteration 2691/ 3814 | consumed samples: 1377792 | consumed tokens: 2821718016 | elapsed time per iteration (s): 5.44 | learning rate: 3.149E-04 | global batch size: 512 | lm loss: 3.132754E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.150 | TFLOPs: 71.11 | +[default7]: iteration 2692/ 3814 | consumed samples: 1378304 | consumed tokens: 2822766592 | elapsed time per iteration (s): 5.42 | learning rate: 3.146E-04 | global batch size: 512 | lm loss: 3.142133E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.397 | TFLOPs: 71.30 | +[default7]: iteration 2693/ 3814 | consumed samples: 1378816 | consumed tokens: 2823815168 | elapsed time per iteration (s): 5.43 | learning rate: 3.142E-04 | global batch size: 512 | lm loss: 3.143346E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 2694/ 3814 | consumed samples: 1379328 | consumed tokens: 2824863744 | elapsed time per iteration (s): 5.42 | learning rate: 3.139E-04 | global batch size: 512 | lm loss: 3.148939E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.393 | TFLOPs: 71.30 | +[default7]: iteration 2695/ 3814 | consumed samples: 1379840 | consumed tokens: 2825912320 | elapsed time per iteration (s): 5.43 | learning rate: 3.135E-04 | global batch size: 512 | lm loss: 3.160905E+00 | loss scale: 131072.0 | grad norm: 0.099 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.317 | TFLOPs: 71.24 | +[default7]: iteration 2696/ 3814 | consumed samples: 1380352 | consumed tokens: 2826960896 | elapsed time per iteration (s): 5.41 | learning rate: 3.132E-04 | global batch size: 512 | lm loss: 3.174545E+00 | loss scale: 131072.0 | grad norm: 0.100 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.592 | TFLOPs: 71.45 | +[default7]: iteration 2697/ 3814 | consumed samples: 1380864 | consumed tokens: 2828009472 | elapsed time per iteration (s): 5.41 | learning rate: 3.128E-04 | global batch size: 512 | lm loss: 3.134090E+00 | loss scale: 131072.0 | grad norm: 0.138 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 2698/ 3814 | consumed samples: 1381376 | consumed tokens: 2829058048 | elapsed time per iteration (s): 5.41 | learning rate: 3.125E-04 | global batch size: 512 | lm loss: 3.202989E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.709 | TFLOPs: 71.53 | +[default7]: iteration 2699/ 3814 | consumed samples: 1381888 | consumed tokens: 2830106624 | elapsed time per iteration (s): 5.43 | learning rate: 3.121E-04 | global batch size: 512 | lm loss: 3.165831E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.240 | TFLOPs: 71.18 | +[default7]: iteration 2700/ 3814 | consumed samples: 1382400 | consumed tokens: 2831155200 | elapsed time per iteration (s): 5.42 | learning rate: 3.118E-04 | global batch size: 512 | lm loss: 3.150235E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.526 | TFLOPs: 71.40 | +[default7]: iteration 2701/ 3814 | consumed samples: 1382912 | consumed tokens: 2832203776 | elapsed time per iteration (s): 5.39 | learning rate: 3.115E-04 | global batch size: 512 | lm loss: 3.143305E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.935 | TFLOPs: 71.71 | +[default7]: iteration 2702/ 3814 | consumed samples: 1383424 | consumed tokens: 2833252352 | elapsed time per iteration (s): 5.43 | learning rate: 3.111E-04 | global batch size: 512 | lm loss: 3.162924E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.378 | TFLOPs: 71.28 | +[default7]: iteration 2703/ 3814 | consumed samples: 1383936 | consumed tokens: 2834300928 | elapsed time per iteration (s): 5.42 | learning rate: 3.108E-04 | global batch size: 512 | lm loss: 3.165852E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.537 | TFLOPs: 71.40 | +[default7]: iteration 2704/ 3814 | consumed samples: 1384448 | consumed tokens: 2835349504 | elapsed time per iteration (s): 5.40 | learning rate: 3.104E-04 | global batch size: 512 | lm loss: 3.135815E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.758 | TFLOPs: 71.57 | +[default7]: iteration 2705/ 3814 | consumed samples: 1384960 | consumed tokens: 2836398080 | elapsed time per iteration (s): 5.42 | learning rate: 3.101E-04 | global batch size: 512 | lm loss: 3.134860E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.458 | TFLOPs: 71.35 | +[default7]: iteration 2706/ 3814 | consumed samples: 1385472 | consumed tokens: 2837446656 | elapsed time per iteration (s): 5.40 | learning rate: 3.097E-04 | global batch size: 512 | lm loss: 3.143033E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.818 | TFLOPs: 71.62 | +[default7]: iteration 2707/ 3814 | consumed samples: 1385984 | consumed tokens: 2838495232 | elapsed time per iteration (s): 5.43 | learning rate: 3.094E-04 | global batch size: 512 | lm loss: 3.150812E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.295 | TFLOPs: 71.22 | +[default7]: iteration 2708/ 3814 | consumed samples: 1386496 | consumed tokens: 2839543808 | elapsed time per iteration (s): 5.39 | learning rate: 3.090E-04 | global batch size: 512 | lm loss: 3.132467E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.905 | TFLOPs: 71.68 | +[default7]: iteration 2709/ 3814 | consumed samples: 1387008 | consumed tokens: 2840592384 | elapsed time per iteration (s): 5.40 | learning rate: 3.087E-04 | global batch size: 512 | lm loss: 3.158340E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.813 | TFLOPs: 71.61 | +[default7]: iteration 2710/ 3814 | consumed samples: 1387520 | consumed tokens: 2841640960 | elapsed time per iteration (s): 5.43 | learning rate: 3.083E-04 | global batch size: 512 | lm loss: 3.141134E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.254 | TFLOPs: 71.19 | +[default7]: iteration 2711/ 3814 | consumed samples: 1388032 | consumed tokens: 2842689536 | elapsed time per iteration (s): 5.41 | learning rate: 3.080E-04 | global batch size: 512 | lm loss: 3.176358E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.665 | TFLOPs: 71.50 | +[default7]: iteration 2712/ 3814 | consumed samples: 1388544 | consumed tokens: 2843738112 | elapsed time per iteration (s): 5.40 | learning rate: 3.076E-04 | global batch size: 512 | lm loss: 3.150558E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.732 | TFLOPs: 71.55 | +[default7]: iteration 2713/ 3814 | consumed samples: 1389056 | consumed tokens: 2844786688 | elapsed time per iteration (s): 5.40 | learning rate: 3.073E-04 | global batch size: 512 | lm loss: 3.130316E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.848 | TFLOPs: 71.64 | +[default7]: iteration 2714/ 3814 | consumed samples: 1389568 | consumed tokens: 2845835264 | elapsed time per iteration (s): 5.41 | learning rate: 3.070E-04 | global batch size: 512 | lm loss: 3.178727E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.618 | TFLOPs: 71.47 | +[default7]: iteration 2715/ 3814 | consumed samples: 1390080 | consumed tokens: 2846883840 | elapsed time per iteration (s): 5.41 | learning rate: 3.066E-04 | global batch size: 512 | lm loss: 3.135706E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.598 | TFLOPs: 71.45 | +[default7]: iteration 2716/ 3814 | consumed samples: 1390592 | consumed tokens: 2847932416 | elapsed time per iteration (s): 5.41 | learning rate: 3.063E-04 | global batch size: 512 | lm loss: 3.153498E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.673 | TFLOPs: 71.51 | +[default7]: iteration 2717/ 3814 | consumed samples: 1391104 | consumed tokens: 2848980992 | elapsed time per iteration (s): 5.41 | learning rate: 3.059E-04 | global batch size: 512 | lm loss: 3.129688E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.610 | TFLOPs: 71.46 | +[default7]: iteration 2718/ 3814 | consumed samples: 1391616 | consumed tokens: 2850029568 | elapsed time per iteration (s): 5.40 | learning rate: 3.056E-04 | global batch size: 512 | lm loss: 3.119632E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.895 | TFLOPs: 71.68 | +[default7]: iteration 2719/ 3814 | consumed samples: 1392128 | consumed tokens: 2851078144 | elapsed time per iteration (s): 5.39 | learning rate: 3.052E-04 | global batch size: 512 | lm loss: 3.142435E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.910 | TFLOPs: 71.69 | +[default7]: iteration 2720/ 3814 | consumed samples: 1392640 | consumed tokens: 2852126720 | elapsed time per iteration (s): 5.41 | learning rate: 3.049E-04 | global batch size: 512 | lm loss: 3.124533E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.625 | TFLOPs: 71.47 | +[default7]: iteration 2721/ 3814 | consumed samples: 1393152 | consumed tokens: 2853175296 | elapsed time per iteration (s): 5.42 | learning rate: 3.046E-04 | global batch size: 512 | lm loss: 3.170887E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.478 | TFLOPs: 71.36 | +[default7]: iteration 2722/ 3814 | consumed samples: 1393664 | consumed tokens: 2854223872 | elapsed time per iteration (s): 5.41 | learning rate: 3.042E-04 | global batch size: 512 | lm loss: 3.106664E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.573 | TFLOPs: 71.43 | +[default7]: iteration 2723/ 3814 | consumed samples: 1394176 | consumed tokens: 2855272448 | elapsed time per iteration (s): 5.44 | learning rate: 3.039E-04 | global batch size: 512 | lm loss: 3.164481E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.176 | TFLOPs: 71.13 | +[default7]: iteration 2724/ 3814 | consumed samples: 1394688 | consumed tokens: 2856321024 | elapsed time per iteration (s): 5.41 | learning rate: 3.035E-04 | global batch size: 512 | lm loss: 3.113247E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.677 | TFLOPs: 71.51 | +[default7]: iteration 2725/ 3814 | consumed samples: 1395200 | consumed tokens: 2857369600 | elapsed time per iteration (s): 5.41 | learning rate: 3.032E-04 | global batch size: 512 | lm loss: 3.130135E+00 | loss scale: 131072.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.725 | TFLOPs: 71.55 | +[default7]: iteration 2726/ 3814 | consumed samples: 1395712 | consumed tokens: 2858418176 | elapsed time per iteration (s): 5.41 | learning rate: 3.028E-04 | global batch size: 512 | lm loss: 3.142535E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.578 | TFLOPs: 71.44 | +[default7]: iteration 2727/ 3814 | consumed samples: 1396224 | consumed tokens: 2859466752 | elapsed time per iteration (s): 5.42 | learning rate: 3.025E-04 | global batch size: 512 | lm loss: 3.138699E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.525 | TFLOPs: 71.40 | +[default7]: iteration 2728/ 3814 | consumed samples: 1396736 | consumed tokens: 2860515328 | elapsed time per iteration (s): 5.41 | learning rate: 3.022E-04 | global batch size: 512 | lm loss: 3.136483E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.585 | TFLOPs: 71.44 | +[default7]: iteration 2729/ 3814 | consumed samples: 1397248 | consumed tokens: 2861563904 | elapsed time per iteration (s): 5.40 | learning rate: 3.018E-04 | global batch size: 512 | lm loss: 3.137189E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.757 | TFLOPs: 71.57 | +[default7]: iteration 2730/ 3814 | consumed samples: 1397760 | consumed tokens: 2862612480 | elapsed time per iteration (s): 5.41 | learning rate: 3.015E-04 | global batch size: 512 | lm loss: 3.158809E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.606 | TFLOPs: 71.46 | +[default7]: iteration 2731/ 3814 | consumed samples: 1398272 | consumed tokens: 2863661056 | elapsed time per iteration (s): 5.41 | learning rate: 3.011E-04 | global batch size: 512 | lm loss: 3.133915E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.622 | TFLOPs: 71.47 | +[default7]: iteration 2732/ 3814 | consumed samples: 1398784 | consumed tokens: 2864709632 | elapsed time per iteration (s): 5.42 | learning rate: 3.008E-04 | global batch size: 512 | lm loss: 3.128614E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.445 | TFLOPs: 71.34 | +[default7]: iteration 2733/ 3814 | consumed samples: 1399296 | consumed tokens: 2865758208 | elapsed time per iteration (s): 5.42 | learning rate: 3.005E-04 | global batch size: 512 | lm loss: 3.150208E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.394 | TFLOPs: 71.30 | +[default7]: iteration 2734/ 3814 | consumed samples: 1399808 | consumed tokens: 2866806784 | elapsed time per iteration (s): 5.43 | learning rate: 3.001E-04 | global batch size: 512 | lm loss: 3.131578E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.352 | TFLOPs: 71.27 | +[default7]: iteration 2735/ 3814 | consumed samples: 1400320 | consumed tokens: 2867855360 | elapsed time per iteration (s): 5.44 | learning rate: 2.998E-04 | global batch size: 512 | lm loss: 3.140397E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.177 | TFLOPs: 71.13 | +[default7]: iteration 2736/ 3814 | consumed samples: 1400832 | consumed tokens: 2868903936 | elapsed time per iteration (s): 5.42 | learning rate: 2.994E-04 | global batch size: 512 | lm loss: 3.174611E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.428 | TFLOPs: 71.32 | +[default7]: iteration 2737/ 3814 | consumed samples: 1401344 | consumed tokens: 2869952512 | elapsed time per iteration (s): 5.40 | learning rate: 2.991E-04 | global batch size: 512 | lm loss: 3.129994E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.823 | TFLOPs: 71.62 | +[default7]: iteration 2738/ 3814 | consumed samples: 1401856 | consumed tokens: 2871001088 | elapsed time per iteration (s): 5.41 | learning rate: 2.988E-04 | global batch size: 512 | lm loss: 3.143082E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.716 | TFLOPs: 71.54 | +[default7]: iteration 2739/ 3814 | consumed samples: 1402368 | consumed tokens: 2872049664 | elapsed time per iteration (s): 5.40 | learning rate: 2.984E-04 | global batch size: 512 | lm loss: 3.149626E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.792 | TFLOPs: 71.60 | +[default7]: iteration 2740/ 3814 | consumed samples: 1402880 | consumed tokens: 2873098240 | elapsed time per iteration (s): 5.41 | learning rate: 2.981E-04 | global batch size: 512 | lm loss: 3.143169E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.651 | TFLOPs: 71.49 | +[default7]: iteration 2741/ 3814 | consumed samples: 1403392 | consumed tokens: 2874146816 | elapsed time per iteration (s): 5.41 | learning rate: 2.977E-04 | global batch size: 512 | lm loss: 3.117866E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.588 | TFLOPs: 71.44 | +[default7]: iteration 2742/ 3814 | consumed samples: 1403904 | consumed tokens: 2875195392 | elapsed time per iteration (s): 5.42 | learning rate: 2.974E-04 | global batch size: 512 | lm loss: 3.143971E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.388 | TFLOPs: 71.29 | +[default7]: iteration 2743/ 3814 | consumed samples: 1404416 | consumed tokens: 2876243968 | elapsed time per iteration (s): 5.42 | learning rate: 2.971E-04 | global batch size: 512 | lm loss: 3.094823E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.398 | TFLOPs: 71.30 | +[default7]: iteration 2744/ 3814 | consumed samples: 1404928 | consumed tokens: 2877292544 | elapsed time per iteration (s): 5.42 | learning rate: 2.967E-04 | global batch size: 512 | lm loss: 3.138594E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.414 | TFLOPs: 71.31 | +[default7]: iteration 2745/ 3814 | consumed samples: 1405440 | consumed tokens: 2878341120 | elapsed time per iteration (s): 5.49 | learning rate: 2.964E-04 | global batch size: 512 | lm loss: 3.143064E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.264 | TFLOPs: 70.44 | +[default7]: iteration 2746/ 3814 | consumed samples: 1405952 | consumed tokens: 2879389696 | elapsed time per iteration (s): 5.43 | learning rate: 2.961E-04 | global batch size: 512 | lm loss: 3.128494E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.307 | TFLOPs: 71.23 | +[default7]: iteration 2747/ 3814 | consumed samples: 1406464 | consumed tokens: 2880438272 | elapsed time per iteration (s): 5.44 | learning rate: 2.957E-04 | global batch size: 512 | lm loss: 3.156925E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.198 | TFLOPs: 71.15 | +[default7]: iteration 2748/ 3814 | consumed samples: 1406976 | consumed tokens: 2881486848 | elapsed time per iteration (s): 5.43 | learning rate: 2.954E-04 | global batch size: 512 | lm loss: 3.147470E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.319 | TFLOPs: 71.24 | +[default7]: iteration 2749/ 3814 | consumed samples: 1407488 | consumed tokens: 2882535424 | elapsed time per iteration (s): 5.42 | learning rate: 2.950E-04 | global batch size: 512 | lm loss: 3.144345E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.438 | TFLOPs: 71.33 | +[default7]: iteration 2750/ 3814 | consumed samples: 1408000 | consumed tokens: 2883584000 | elapsed time per iteration (s): 5.42 | learning rate: 2.947E-04 | global batch size: 512 | lm loss: 3.133523E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.475 | TFLOPs: 71.36 | +[default1]:[2023-02-16 10:28:22,330] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_01-model_01-model_states.pt... +[default0]:saving checkpoint at iteration 2750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 10:28:22,327] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step2750 is begin to save! +[default0]:[2023-02-16 10:28:22,330] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,481] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,481] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,483] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,483] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_04-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,537] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,538] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,540] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,540] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,595] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,595] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:22,647] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,647] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,605] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,606] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:22,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,708] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,708] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:22,763] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,764] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,727] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,728] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:22,782] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,782] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,816] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,817] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:22,867] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,868] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:22,890] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,891] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:22,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,922] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_12-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:22,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:22,981] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:22,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:22,944] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,037] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,006] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,062] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,063] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,089] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,143] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,118] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,118] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,171] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,197] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,197] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,250] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,225] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,225] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_17-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,289] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,289] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,314] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,370] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,370] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,347] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,347] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_19-model_00-model_states.pt. +[default1]:[2023-02-16 10:28:23,424] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,424] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,477] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,478] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,403] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_20-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,456] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,530] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,531] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_23-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,586] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,586] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,509] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,509] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,563] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,563] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,641] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,619] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,620] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_24-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,675] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,676] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,693] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,693] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_26-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,749] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,750] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,728] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,729] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_26-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,786] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,787] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,810] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_28-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:23,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,870] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,857] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,858] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,930] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_30-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:23,913] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,914] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_29-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:23,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:23,969] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_30-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:23,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:23,995] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_31-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:24,047] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,047] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:24,031] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,032] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_31-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:24,095] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,095] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:24,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,107] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_33-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:24,158] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,158] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_34-model_01-model_states.pt... +[default0]:[2023-02-16 10:28:24,148] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,149] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 10:28:24,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,213] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_35-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:24,266] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,267] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 10:28:24,268] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 10:28:24,269] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_01_model_states.pt +[default1]:[2023-02-16 10:28:24,269] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 10:28:24,271] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_01_model_states.pt. +[default0]:[2023-02-16 10:28:24,204] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,204] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_34-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:24,260] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,261] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_35-model_00-model_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:28:24,312] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,313] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 10:28:24,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 10:28:24,315] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_00_model_states.pt +[default0]:[2023-02-16 10:28:24,315] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 10:28:24,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:28:24,348] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:28:24,856] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:24,856] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:24,856] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:24,838] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:24,839] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:24,839] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:24,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:24,870] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:24,870] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:24,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:24,881] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:24,881] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:24,880] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:24,880] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:24,880] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:24,921] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:24,921] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:24,921] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:24,953] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:24,953] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:24,953] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:24,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:24,937] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:24,937] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:24,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:24,957] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:24,957] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:24,953] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:24,954] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:24,954] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:24,932] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:24,932] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:24,932] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:24,987] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:24,988] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:24,988] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:24,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:24,928] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:24,928] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:24,993] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:24,993] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:24,993] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:24,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:24,954] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:24,954] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:24,977] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:24,978] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:24,978] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:24,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:24,997] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:24,997] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:24,986] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:24,987] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:24,987] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:25,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:25,024] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:25,024] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:24,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:24,935] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:24,935] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:25,006] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,006] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,006] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:24,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:24,955] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:24,955] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,020] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,020] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:25,047] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:25,047] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:25,047] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:25,027] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:25,027] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:25,027] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:25,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:25,026] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:25,026] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,028] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,028] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:24,957] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:24,957] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:24,957] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,005] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,006] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,006] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:24,967] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:24,967] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:24,967] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,019] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,019] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,063] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,063] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:24,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:24,994] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:24,994] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:24,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:24,981] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:24,981] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:25,045] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:25,045] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:25,045] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:25,040] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:25,040] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:25,040] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,051] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,051] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:25,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:25,088] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:25,088] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:25,069] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:25,069] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:25,069] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:25,058] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,058] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,058] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:25,051] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:25,051] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:25,051] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:25,089] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,089] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,090] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:25,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:25,024] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:25,024] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:25,063] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,063] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,063] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,044] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,044] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,044] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:25,092] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:25,092] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:25,092] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:25,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:25,072] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:25,072] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:25,036] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:25,036] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:25,036] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,056] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,056] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,056] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:25,079] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:25,079] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:25,079] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default5]:[2023-02-16 10:28:25,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,060] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,060] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default2]:[2023-02-16 10:28:25,128] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:28:25,129] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:28:25,129] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:25,072] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:25,072] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:25,073] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default1]:[2023-02-16 10:28:25,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:28:25,121] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:28:25,121] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:25,066] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:25,066] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:25,066] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,121] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,121] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,121] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default3]:[2023-02-16 10:28:25,088] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:28:25,088] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:28:25,088] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default4]:[2023-02-16 10:28:25,075] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:28:25,075] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:28:25,075] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:25,099] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:25,099] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:25,099] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]:[2023-02-16 10:28:25,162] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:28:25,162] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:28:25,162] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,188] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,188] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,188] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:[2023-02-16 10:28:25,231] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:28:25,231] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:28:25,231] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default6]:[2023-02-16 10:28:25,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:28:25,235] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:28:25,235] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default0]: successfully saved checkpoint at iteration 2750 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default5]:[2023-02-16 10:28:25,255] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:28:25,255] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step2750/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:28:25,255] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step2750 is ready now! +[default7]:time (ms) | save-checkpoint: 2929.07 +[default7]: iteration 2751/ 3814 | consumed samples: 1408512 | consumed tokens: 2884632576 | elapsed time per iteration (s): 8.39 | learning rate: 2.944E-04 | global batch size: 512 | lm loss: 3.134408E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 61.045 | TFLOPs: 46.11 | +[default7]: iteration 2752/ 3814 | consumed samples: 1409024 | consumed tokens: 2885681152 | elapsed time per iteration (s): 5.42 | learning rate: 2.940E-04 | global batch size: 512 | lm loss: 3.140137E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.397 | TFLOPs: 71.30 | +[default7]: iteration 2753/ 3814 | consumed samples: 1409536 | consumed tokens: 2886729728 | elapsed time per iteration (s): 5.43 | learning rate: 2.937E-04 | global batch size: 512 | lm loss: 3.135284E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.305 | TFLOPs: 71.23 | +[default7]: iteration 2754/ 3814 | consumed samples: 1410048 | consumed tokens: 2887778304 | elapsed time per iteration (s): 5.43 | learning rate: 2.934E-04 | global batch size: 512 | lm loss: 3.149498E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.277 | TFLOPs: 71.21 | +[default7]: iteration 2755/ 3814 | consumed samples: 1410560 | consumed tokens: 2888826880 | elapsed time per iteration (s): 5.41 | learning rate: 2.930E-04 | global batch size: 512 | lm loss: 3.122391E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.725 | TFLOPs: 71.55 | +[default7]: iteration 2756/ 3814 | consumed samples: 1411072 | consumed tokens: 2889875456 | elapsed time per iteration (s): 5.40 | learning rate: 2.927E-04 | global batch size: 512 | lm loss: 3.123090E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.784 | TFLOPs: 71.59 | +[default7]: iteration 2757/ 3814 | consumed samples: 1411584 | consumed tokens: 2890924032 | elapsed time per iteration (s): 5.42 | learning rate: 2.923E-04 | global batch size: 512 | lm loss: 3.142074E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.519 | TFLOPs: 71.39 | +[default7]: iteration 2758/ 3814 | consumed samples: 1412096 | consumed tokens: 2891972608 | elapsed time per iteration (s): 5.40 | learning rate: 2.920E-04 | global batch size: 512 | lm loss: 3.104448E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.887 | TFLOPs: 71.67 | +[default7]: iteration 2759/ 3814 | consumed samples: 1412608 | consumed tokens: 2893021184 | elapsed time per iteration (s): 5.38 | learning rate: 2.917E-04 | global batch size: 512 | lm loss: 3.167380E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.097 | TFLOPs: 71.83 | +[default7]: iteration 2760/ 3814 | consumed samples: 1413120 | consumed tokens: 2894069760 | elapsed time per iteration (s): 5.40 | learning rate: 2.913E-04 | global batch size: 512 | lm loss: 3.130821E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 2761/ 3814 | consumed samples: 1413632 | consumed tokens: 2895118336 | elapsed time per iteration (s): 5.42 | learning rate: 2.910E-04 | global batch size: 512 | lm loss: 3.154698E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.456 | TFLOPs: 71.34 | +[default7]: iteration 2762/ 3814 | consumed samples: 1414144 | consumed tokens: 2896166912 | elapsed time per iteration (s): 5.42 | learning rate: 2.907E-04 | global batch size: 512 | lm loss: 3.150602E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.483 | TFLOPs: 71.36 | +[default7]: iteration 2763/ 3814 | consumed samples: 1414656 | consumed tokens: 2897215488 | elapsed time per iteration (s): 5.40 | learning rate: 2.903E-04 | global batch size: 512 | lm loss: 3.111335E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.65 | +[default7]: iteration 2764/ 3814 | consumed samples: 1415168 | consumed tokens: 2898264064 | elapsed time per iteration (s): 5.43 | learning rate: 2.900E-04 | global batch size: 512 | lm loss: 3.140113E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.249 | TFLOPs: 71.19 | +[default7]: iteration 2765/ 3814 | consumed samples: 1415680 | consumed tokens: 2899312640 | elapsed time per iteration (s): 5.40 | learning rate: 2.897E-04 | global batch size: 512 | lm loss: 3.122486E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.746 | TFLOPs: 71.56 | +[default7]: iteration 2766/ 3814 | consumed samples: 1416192 | consumed tokens: 2900361216 | elapsed time per iteration (s): 5.40 | learning rate: 2.893E-04 | global batch size: 512 | lm loss: 3.126340E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.743 | TFLOPs: 71.56 | +[default7]: iteration 2767/ 3814 | consumed samples: 1416704 | consumed tokens: 2901409792 | elapsed time per iteration (s): 5.41 | learning rate: 2.890E-04 | global batch size: 512 | lm loss: 3.129195E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.596 | TFLOPs: 71.45 | +[default7]: iteration 2768/ 3814 | consumed samples: 1417216 | consumed tokens: 2902458368 | elapsed time per iteration (s): 5.41 | learning rate: 2.887E-04 | global batch size: 512 | lm loss: 3.158066E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.635 | TFLOPs: 71.48 | +[default7]: iteration 2769/ 3814 | consumed samples: 1417728 | consumed tokens: 2903506944 | elapsed time per iteration (s): 5.40 | learning rate: 2.883E-04 | global batch size: 512 | lm loss: 3.118733E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.861 | TFLOPs: 71.65 | +[default7]: iteration 2770/ 3814 | consumed samples: 1418240 | consumed tokens: 2904555520 | elapsed time per iteration (s): 5.42 | learning rate: 2.880E-04 | global batch size: 512 | lm loss: 3.117269E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.380 | TFLOPs: 71.29 | +[default7]: iteration 2771/ 3814 | consumed samples: 1418752 | consumed tokens: 2905604096 | elapsed time per iteration (s): 5.42 | learning rate: 2.877E-04 | global batch size: 512 | lm loss: 3.149072E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.435 | TFLOPs: 71.33 | +[default7]: iteration 2772/ 3814 | consumed samples: 1419264 | consumed tokens: 2906652672 | elapsed time per iteration (s): 5.42 | learning rate: 2.873E-04 | global batch size: 512 | lm loss: 3.122150E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.545 | TFLOPs: 71.41 | +[default7]: iteration 2773/ 3814 | consumed samples: 1419776 | consumed tokens: 2907701248 | elapsed time per iteration (s): 5.41 | learning rate: 2.870E-04 | global batch size: 512 | lm loss: 3.154816E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.680 | TFLOPs: 71.51 | +[default7]: iteration 2774/ 3814 | consumed samples: 1420288 | consumed tokens: 2908749824 | elapsed time per iteration (s): 5.39 | learning rate: 2.867E-04 | global batch size: 512 | lm loss: 3.146656E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.984 | TFLOPs: 71.74 | +[default7]: iteration 2775/ 3814 | consumed samples: 1420800 | consumed tokens: 2909798400 | elapsed time per iteration (s): 5.40 | learning rate: 2.863E-04 | global batch size: 512 | lm loss: 3.105781E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.849 | TFLOPs: 71.64 | +[default7]: iteration 2776/ 3814 | consumed samples: 1421312 | consumed tokens: 2910846976 | elapsed time per iteration (s): 5.41 | learning rate: 2.860E-04 | global batch size: 512 | lm loss: 3.129378E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 2777/ 3814 | consumed samples: 1421824 | consumed tokens: 2911895552 | elapsed time per iteration (s): 5.41 | learning rate: 2.857E-04 | global batch size: 512 | lm loss: 3.134856E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.688 | TFLOPs: 71.52 | +[default7]: iteration 2778/ 3814 | consumed samples: 1422336 | consumed tokens: 2912944128 | elapsed time per iteration (s): 5.40 | learning rate: 2.854E-04 | global batch size: 512 | lm loss: 3.163314E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.877 | TFLOPs: 71.66 | +[default7]: iteration 2779/ 3814 | consumed samples: 1422848 | consumed tokens: 2913992704 | elapsed time per iteration (s): 5.40 | learning rate: 2.850E-04 | global batch size: 512 | lm loss: 3.134766E+00 | loss scale: 131072.0 | grad norm: 0.136 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.852 | TFLOPs: 71.64 | +[default7]: iteration 2780/ 3814 | consumed samples: 1423360 | consumed tokens: 2915041280 | elapsed time per iteration (s): 5.41 | learning rate: 2.847E-04 | global batch size: 512 | lm loss: 3.142942E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.654 | TFLOPs: 71.49 | +[default7]: iteration 2781/ 3814 | consumed samples: 1423872 | consumed tokens: 2916089856 | elapsed time per iteration (s): 5.41 | learning rate: 2.844E-04 | global batch size: 512 | lm loss: 3.149663E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.671 | TFLOPs: 71.51 | +[default7]: iteration 2782/ 3814 | consumed samples: 1424384 | consumed tokens: 2917138432 | elapsed time per iteration (s): 5.41 | learning rate: 2.840E-04 | global batch size: 512 | lm loss: 3.104757E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.698 | TFLOPs: 71.53 | +[default7]: iteration 2783/ 3814 | consumed samples: 1424896 | consumed tokens: 2918187008 | elapsed time per iteration (s): 5.42 | learning rate: 2.837E-04 | global batch size: 512 | lm loss: 3.095318E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.523 | TFLOPs: 71.39 | +[default7]: iteration 2784/ 3814 | consumed samples: 1425408 | consumed tokens: 2919235584 | elapsed time per iteration (s): 5.42 | learning rate: 2.834E-04 | global batch size: 512 | lm loss: 3.123021E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.485 | TFLOPs: 71.37 | +[default7]: iteration 2785/ 3814 | consumed samples: 1425920 | consumed tokens: 2920284160 | elapsed time per iteration (s): 5.41 | learning rate: 2.830E-04 | global batch size: 512 | lm loss: 3.130916E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.628 | TFLOPs: 71.47 | +[default7]: iteration 2786/ 3814 | consumed samples: 1426432 | consumed tokens: 2921332736 | elapsed time per iteration (s): 5.41 | learning rate: 2.827E-04 | global batch size: 512 | lm loss: 3.138022E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.574 | TFLOPs: 71.43 | +[default7]: iteration 2787/ 3814 | consumed samples: 1426944 | consumed tokens: 2922381312 | elapsed time per iteration (s): 5.41 | learning rate: 2.824E-04 | global batch size: 512 | lm loss: 3.123765E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.660 | TFLOPs: 71.50 | +[default7]: iteration 2788/ 3814 | consumed samples: 1427456 | consumed tokens: 2923429888 | elapsed time per iteration (s): 5.42 | learning rate: 2.821E-04 | global batch size: 512 | lm loss: 3.125536E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.462 | TFLOPs: 71.35 | +[default7]: iteration 2789/ 3814 | consumed samples: 1427968 | consumed tokens: 2924478464 | elapsed time per iteration (s): 5.43 | learning rate: 2.817E-04 | global batch size: 512 | lm loss: 3.159559E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.302 | TFLOPs: 71.23 | +[default7]: iteration 2790/ 3814 | consumed samples: 1428480 | consumed tokens: 2925527040 | elapsed time per iteration (s): 5.41 | learning rate: 2.814E-04 | global batch size: 512 | lm loss: 3.117987E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.649 | TFLOPs: 71.49 | +[default7]: iteration 2791/ 3814 | consumed samples: 1428992 | consumed tokens: 2926575616 | elapsed time per iteration (s): 5.42 | learning rate: 2.811E-04 | global batch size: 512 | lm loss: 3.152164E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.549 | TFLOPs: 71.41 | +[default7]: iteration 2792/ 3814 | consumed samples: 1429504 | consumed tokens: 2927624192 | elapsed time per iteration (s): 5.43 | learning rate: 2.807E-04 | global batch size: 512 | lm loss: 3.122350E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.295 | TFLOPs: 71.22 | +[default7]: iteration 2793/ 3814 | consumed samples: 1430016 | consumed tokens: 2928672768 | elapsed time per iteration (s): 5.41 | learning rate: 2.804E-04 | global batch size: 512 | lm loss: 3.134854E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.599 | TFLOPs: 71.45 | +[default7]: iteration 2794/ 3814 | consumed samples: 1430528 | consumed tokens: 2929721344 | elapsed time per iteration (s): 5.43 | learning rate: 2.801E-04 | global batch size: 512 | lm loss: 3.175410E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.342 | TFLOPs: 71.26 | +[default7]: iteration 2795/ 3814 | consumed samples: 1431040 | consumed tokens: 2930769920 | elapsed time per iteration (s): 5.42 | learning rate: 2.798E-04 | global batch size: 512 | lm loss: 3.106360E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.485 | TFLOPs: 71.37 | +[default7]: iteration 2796/ 3814 | consumed samples: 1431552 | consumed tokens: 2931818496 | elapsed time per iteration (s): 5.41 | learning rate: 2.794E-04 | global batch size: 512 | lm loss: 3.109015E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.631 | TFLOPs: 71.48 | +[default7]: iteration 2797/ 3814 | consumed samples: 1432064 | consumed tokens: 2932867072 | elapsed time per iteration (s): 5.41 | learning rate: 2.791E-04 | global batch size: 512 | lm loss: 3.120878E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.684 | TFLOPs: 71.52 | +[default7]: iteration 2798/ 3814 | consumed samples: 1432576 | consumed tokens: 2933915648 | elapsed time per iteration (s): 5.41 | learning rate: 2.788E-04 | global batch size: 512 | lm loss: 3.115105E+00 | loss scale: 131072.0 | grad norm: 0.131 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 2799/ 3814 | consumed samples: 1433088 | consumed tokens: 2934964224 | elapsed time per iteration (s): 5.41 | learning rate: 2.785E-04 | global batch size: 512 | lm loss: 3.143191E+00 | loss scale: 131072.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.607 | TFLOPs: 71.46 | +[default7]: iteration 2800/ 3814 | consumed samples: 1433600 | consumed tokens: 2936012800 | elapsed time per iteration (s): 5.42 | learning rate: 2.781E-04 | global batch size: 512 | lm loss: 3.140828E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.530 | TFLOPs: 71.40 | +[default7]: iteration 2801/ 3814 | consumed samples: 1434112 | consumed tokens: 2937061376 | elapsed time per iteration (s): 5.43 | learning rate: 2.778E-04 | global batch size: 512 | lm loss: 3.111793E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.303 | TFLOPs: 71.23 | +[default7]: iteration 2802/ 3814 | consumed samples: 1434624 | consumed tokens: 2938109952 | elapsed time per iteration (s): 5.43 | learning rate: 2.775E-04 | global batch size: 512 | lm loss: 3.127179E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.299 | TFLOPs: 71.22 | +[default7]: iteration 2803/ 3814 | consumed samples: 1435136 | consumed tokens: 2939158528 | elapsed time per iteration (s): 5.43 | learning rate: 2.772E-04 | global batch size: 512 | lm loss: 3.144023E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.310 | TFLOPs: 71.23 | +[default7]: iteration 2804/ 3814 | consumed samples: 1435648 | consumed tokens: 2940207104 | elapsed time per iteration (s): 5.43 | learning rate: 2.768E-04 | global batch size: 512 | lm loss: 3.136039E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.268 | TFLOPs: 71.20 | +[default7]: iteration 2805/ 3814 | consumed samples: 1436160 | consumed tokens: 2941255680 | elapsed time per iteration (s): 5.44 | learning rate: 2.765E-04 | global batch size: 512 | lm loss: 3.125427E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.074 | TFLOPs: 71.06 | +[default7]: iteration 2806/ 3814 | consumed samples: 1436672 | consumed tokens: 2942304256 | elapsed time per iteration (s): 5.43 | learning rate: 2.762E-04 | global batch size: 512 | lm loss: 3.101315E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.265 | TFLOPs: 71.20 | +[default7]: iteration 2807/ 3814 | consumed samples: 1437184 | consumed tokens: 2943352832 | elapsed time per iteration (s): 5.42 | learning rate: 2.759E-04 | global batch size: 512 | lm loss: 3.115948E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.389 | TFLOPs: 71.29 | +[default7]: iteration 2808/ 3814 | consumed samples: 1437696 | consumed tokens: 2944401408 | elapsed time per iteration (s): 5.42 | learning rate: 2.755E-04 | global batch size: 512 | lm loss: 3.123845E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.467 | TFLOPs: 71.35 | +[default7]: iteration 2809/ 3814 | consumed samples: 1438208 | consumed tokens: 2945449984 | elapsed time per iteration (s): 5.44 | learning rate: 2.752E-04 | global batch size: 512 | lm loss: 3.123178E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.130 | TFLOPs: 71.10 | +[default7]: iteration 2810/ 3814 | consumed samples: 1438720 | consumed tokens: 2946498560 | elapsed time per iteration (s): 5.41 | learning rate: 2.749E-04 | global batch size: 512 | lm loss: 3.120574E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.719 | TFLOPs: 71.54 | +[default7]: iteration 2811/ 3814 | consumed samples: 1439232 | consumed tokens: 2947547136 | elapsed time per iteration (s): 5.38 | learning rate: 2.746E-04 | global batch size: 512 | lm loss: 3.117220E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.141 | TFLOPs: 71.86 | +[default7]: iteration 2812/ 3814 | consumed samples: 1439744 | consumed tokens: 2948595712 | elapsed time per iteration (s): 5.41 | learning rate: 2.742E-04 | global batch size: 512 | lm loss: 3.147072E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.712 | TFLOPs: 71.54 | +[default7]: iteration 2813/ 3814 | consumed samples: 1440256 | consumed tokens: 2949644288 | elapsed time per iteration (s): 5.40 | learning rate: 2.739E-04 | global batch size: 512 | lm loss: 3.142767E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.846 | TFLOPs: 71.64 | +[default7]: iteration 2814/ 3814 | consumed samples: 1440768 | consumed tokens: 2950692864 | elapsed time per iteration (s): 5.38 | learning rate: 2.736E-04 | global batch size: 512 | lm loss: 3.131826E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.095 | TFLOPs: 71.83 | +[default7]: iteration 2815/ 3814 | consumed samples: 1441280 | consumed tokens: 2951741440 | elapsed time per iteration (s): 5.37 | learning rate: 2.733E-04 | global batch size: 512 | lm loss: 3.155791E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.272 | TFLOPs: 71.96 | +[default7]: iteration 2816/ 3814 | consumed samples: 1441792 | consumed tokens: 2952790016 | elapsed time per iteration (s): 5.40 | learning rate: 2.729E-04 | global batch size: 512 | lm loss: 3.103438E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.796 | TFLOPs: 71.60 | +[default7]: iteration 2817/ 3814 | consumed samples: 1442304 | consumed tokens: 2953838592 | elapsed time per iteration (s): 5.43 | learning rate: 2.726E-04 | global batch size: 512 | lm loss: 3.112535E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.220 | TFLOPs: 71.17 | +[default7]: iteration 2818/ 3814 | consumed samples: 1442816 | consumed tokens: 2954887168 | elapsed time per iteration (s): 5.42 | learning rate: 2.723E-04 | global batch size: 512 | lm loss: 3.140960E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.450 | TFLOPs: 71.34 | +[default7]: iteration 2819/ 3814 | consumed samples: 1443328 | consumed tokens: 2955935744 | elapsed time per iteration (s): 5.41 | learning rate: 2.720E-04 | global batch size: 512 | lm loss: 3.124899E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.580 | TFLOPs: 71.44 | +[default7]: iteration 2820/ 3814 | consumed samples: 1443840 | consumed tokens: 2956984320 | elapsed time per iteration (s): 5.43 | learning rate: 2.717E-04 | global batch size: 512 | lm loss: 3.123769E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.288 | TFLOPs: 71.22 | +[default7]: iteration 2821/ 3814 | consumed samples: 1444352 | consumed tokens: 2958032896 | elapsed time per iteration (s): 5.44 | learning rate: 2.713E-04 | global batch size: 512 | lm loss: 3.128213E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.178 | TFLOPs: 71.13 | +[default7]: iteration 2822/ 3814 | consumed samples: 1444864 | consumed tokens: 2959081472 | elapsed time per iteration (s): 5.42 | learning rate: 2.710E-04 | global batch size: 512 | lm loss: 3.135195E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.420 | TFLOPs: 71.32 | +[default7]: iteration 2823/ 3814 | consumed samples: 1445376 | consumed tokens: 2960130048 | elapsed time per iteration (s): 5.36 | learning rate: 2.707E-04 | global batch size: 512 | lm loss: 3.126755E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.517 | TFLOPs: 72.15 | +[default7]: iteration 2824/ 3814 | consumed samples: 1445888 | consumed tokens: 2961178624 | elapsed time per iteration (s): 5.37 | learning rate: 2.704E-04 | global batch size: 512 | lm loss: 3.121938E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.351 | TFLOPs: 72.02 | +[default7]: iteration 2825/ 3814 | consumed samples: 1446400 | consumed tokens: 2962227200 | elapsed time per iteration (s): 5.36 | learning rate: 2.701E-04 | global batch size: 512 | lm loss: 3.121275E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.570 | TFLOPs: 72.19 | +[default7]: iteration 2826/ 3814 | consumed samples: 1446912 | consumed tokens: 2963275776 | elapsed time per iteration (s): 5.35 | learning rate: 2.697E-04 | global batch size: 512 | lm loss: 3.116791E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.757 | TFLOPs: 72.33 | +[default7]: iteration 2827/ 3814 | consumed samples: 1447424 | consumed tokens: 2964324352 | elapsed time per iteration (s): 5.36 | learning rate: 2.694E-04 | global batch size: 512 | lm loss: 3.152811E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.562 | TFLOPs: 72.18 | +[default7]: iteration 2828/ 3814 | consumed samples: 1447936 | consumed tokens: 2965372928 | elapsed time per iteration (s): 5.38 | learning rate: 2.691E-04 | global batch size: 512 | lm loss: 3.138495E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.183 | TFLOPs: 71.89 | +[default7]: iteration 2829/ 3814 | consumed samples: 1448448 | consumed tokens: 2966421504 | elapsed time per iteration (s): 5.36 | learning rate: 2.688E-04 | global batch size: 512 | lm loss: 3.141536E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.565 | TFLOPs: 72.18 | +[default7]: iteration 2830/ 3814 | consumed samples: 1448960 | consumed tokens: 2967470080 | elapsed time per iteration (s): 5.35 | learning rate: 2.685E-04 | global batch size: 512 | lm loss: 3.131988E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.705 | TFLOPs: 72.29 | +[default7]: iteration 2831/ 3814 | consumed samples: 1449472 | consumed tokens: 2968518656 | elapsed time per iteration (s): 5.41 | learning rate: 2.681E-04 | global batch size: 512 | lm loss: 3.124980E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.684 | TFLOPs: 71.52 | +[default7]: iteration 2832/ 3814 | consumed samples: 1449984 | consumed tokens: 2969567232 | elapsed time per iteration (s): 5.40 | learning rate: 2.678E-04 | global batch size: 512 | lm loss: 3.122646E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.898 | TFLOPs: 71.68 | +[default7]: iteration 2833/ 3814 | consumed samples: 1450496 | consumed tokens: 2970615808 | elapsed time per iteration (s): 5.36 | learning rate: 2.675E-04 | global batch size: 512 | lm loss: 3.117243E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.550 | TFLOPs: 72.17 | +[default7]: iteration 2834/ 3814 | consumed samples: 1451008 | consumed tokens: 2971664384 | elapsed time per iteration (s): 5.39 | learning rate: 2.672E-04 | global batch size: 512 | lm loss: 3.143202E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.916 | TFLOPs: 71.69 | +[default7]: iteration 2835/ 3814 | consumed samples: 1451520 | consumed tokens: 2972712960 | elapsed time per iteration (s): 5.36 | learning rate: 2.669E-04 | global batch size: 512 | lm loss: 3.117477E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.543 | TFLOPs: 72.16 | +[default7]: iteration 2836/ 3814 | consumed samples: 1452032 | consumed tokens: 2973761536 | elapsed time per iteration (s): 5.42 | learning rate: 2.665E-04 | global batch size: 512 | lm loss: 3.110807E+00 | loss scale: 131072.0 | grad norm: 0.103 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.514 | TFLOPs: 71.39 | +[default7]: iteration 2837/ 3814 | consumed samples: 1452544 | consumed tokens: 2974810112 | elapsed time per iteration (s): 5.39 | learning rate: 2.662E-04 | global batch size: 512 | lm loss: 3.139376E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.056 | TFLOPs: 71.80 | +[default7]: iteration 2838/ 3814 | consumed samples: 1453056 | consumed tokens: 2975858688 | elapsed time per iteration (s): 5.34 | learning rate: 2.659E-04 | global batch size: 512 | lm loss: 3.092124E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.807 | TFLOPs: 72.36 | +[default7]: iteration 2839/ 3814 | consumed samples: 1453568 | consumed tokens: 2976907264 | elapsed time per iteration (s): 5.35 | learning rate: 2.656E-04 | global batch size: 512 | lm loss: 3.121587E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.643 | TFLOPs: 72.24 | +[default7]: iteration 2840/ 3814 | consumed samples: 1454080 | consumed tokens: 2977955840 | elapsed time per iteration (s): 5.35 | learning rate: 2.653E-04 | global batch size: 512 | lm loss: 3.120746E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.723 | TFLOPs: 72.30 | +[default7]: iteration 2841/ 3814 | consumed samples: 1454592 | consumed tokens: 2979004416 | elapsed time per iteration (s): 5.37 | learning rate: 2.650E-04 | global batch size: 512 | lm loss: 3.139601E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.370 | TFLOPs: 72.03 | +[default7]: iteration 2842/ 3814 | consumed samples: 1455104 | consumed tokens: 2980052992 | elapsed time per iteration (s): 5.38 | learning rate: 2.646E-04 | global batch size: 512 | lm loss: 3.132928E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.158 | TFLOPs: 71.87 | +[default7]: iteration 2843/ 3814 | consumed samples: 1455616 | consumed tokens: 2981101568 | elapsed time per iteration (s): 5.39 | learning rate: 2.643E-04 | global batch size: 512 | lm loss: 3.121087E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.029 | TFLOPs: 71.78 | +[default7]: iteration 2844/ 3814 | consumed samples: 1456128 | consumed tokens: 2982150144 | elapsed time per iteration (s): 5.35 | learning rate: 2.640E-04 | global batch size: 512 | lm loss: 3.095918E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.768 | TFLOPs: 72.33 | +[default7]: iteration 2845/ 3814 | consumed samples: 1456640 | consumed tokens: 2983198720 | elapsed time per iteration (s): 5.35 | learning rate: 2.637E-04 | global batch size: 512 | lm loss: 3.116070E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.748 | TFLOPs: 72.32 | +[default7]: iteration 2846/ 3814 | consumed samples: 1457152 | consumed tokens: 2984247296 | elapsed time per iteration (s): 5.37 | learning rate: 2.634E-04 | global batch size: 512 | lm loss: 3.106281E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.333 | TFLOPs: 72.01 | +[default7]: iteration 2847/ 3814 | consumed samples: 1457664 | consumed tokens: 2985295872 | elapsed time per iteration (s): 5.37 | learning rate: 2.631E-04 | global batch size: 512 | lm loss: 3.142737E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.430 | TFLOPs: 72.08 | +[default7]: iteration 2848/ 3814 | consumed samples: 1458176 | consumed tokens: 2986344448 | elapsed time per iteration (s): 5.37 | learning rate: 2.628E-04 | global batch size: 512 | lm loss: 3.142614E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.381 | TFLOPs: 72.04 | +[default7]: iteration 2849/ 3814 | consumed samples: 1458688 | consumed tokens: 2987393024 | elapsed time per iteration (s): 5.36 | learning rate: 2.624E-04 | global batch size: 512 | lm loss: 3.128273E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.487 | TFLOPs: 72.12 | +[default7]: iteration 2850/ 3814 | consumed samples: 1459200 | consumed tokens: 2988441600 | elapsed time per iteration (s): 5.37 | learning rate: 2.621E-04 | global batch size: 512 | lm loss: 3.126122E+00 | loss scale: 131072.0 | grad norm: 0.129 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.407 | TFLOPs: 72.06 | +[default7]: iteration 2851/ 3814 | consumed samples: 1459712 | consumed tokens: 2989490176 | elapsed time per iteration (s): 5.37 | learning rate: 2.618E-04 | global batch size: 512 | lm loss: 3.113184E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.432 | TFLOPs: 72.08 | +[default7]: iteration 2852/ 3814 | consumed samples: 1460224 | consumed tokens: 2990538752 | elapsed time per iteration (s): 5.36 | learning rate: 2.615E-04 | global batch size: 512 | lm loss: 3.153566E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.457 | TFLOPs: 72.10 | +[default7]: iteration 2853/ 3814 | consumed samples: 1460736 | consumed tokens: 2991587328 | elapsed time per iteration (s): 5.37 | learning rate: 2.612E-04 | global batch size: 512 | lm loss: 3.102782E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.432 | TFLOPs: 72.08 | +[default7]: iteration 2854/ 3814 | consumed samples: 1461248 | consumed tokens: 2992635904 | elapsed time per iteration (s): 5.36 | learning rate: 2.609E-04 | global batch size: 512 | lm loss: 3.104898E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.573 | TFLOPs: 72.19 | +[default7]: iteration 2855/ 3814 | consumed samples: 1461760 | consumed tokens: 2993684480 | elapsed time per iteration (s): 5.40 | learning rate: 2.606E-04 | global batch size: 512 | lm loss: 3.115293E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.894 | TFLOPs: 71.67 | +[default7]: iteration 2856/ 3814 | consumed samples: 1462272 | consumed tokens: 2994733056 | elapsed time per iteration (s): 5.36 | learning rate: 2.602E-04 | global batch size: 512 | lm loss: 3.117151E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.489 | TFLOPs: 72.12 | +[default7]: iteration 2857/ 3814 | consumed samples: 1462784 | consumed tokens: 2995781632 | elapsed time per iteration (s): 5.38 | learning rate: 2.599E-04 | global batch size: 512 | lm loss: 3.118134E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.105 | TFLOPs: 71.83 | +[default7]: iteration 2858/ 3814 | consumed samples: 1463296 | consumed tokens: 2996830208 | elapsed time per iteration (s): 5.36 | learning rate: 2.596E-04 | global batch size: 512 | lm loss: 3.119617E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.576 | TFLOPs: 72.19 | +[default7]: iteration 2859/ 3814 | consumed samples: 1463808 | consumed tokens: 2997878784 | elapsed time per iteration (s): 5.40 | learning rate: 2.593E-04 | global batch size: 512 | lm loss: 3.132787E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.819 | TFLOPs: 71.62 | +[default7]: iteration 2860/ 3814 | consumed samples: 1464320 | consumed tokens: 2998927360 | elapsed time per iteration (s): 5.34 | learning rate: 2.590E-04 | global batch size: 512 | lm loss: 3.140940E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.862 | TFLOPs: 72.41 | +[default7]: iteration 2861/ 3814 | consumed samples: 1464832 | consumed tokens: 2999975936 | elapsed time per iteration (s): 5.35 | learning rate: 2.587E-04 | global batch size: 512 | lm loss: 3.142159E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.672 | TFLOPs: 72.26 | +[default7]: iteration 2862/ 3814 | consumed samples: 1465344 | consumed tokens: 3001024512 | elapsed time per iteration (s): 5.35 | learning rate: 2.584E-04 | global batch size: 512 | lm loss: 3.050631E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.619 | TFLOPs: 72.22 | +[default7]: iteration 2863/ 3814 | consumed samples: 1465856 | consumed tokens: 3002073088 | elapsed time per iteration (s): 5.39 | learning rate: 2.581E-04 | global batch size: 512 | lm loss: 3.133455E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.938 | TFLOPs: 71.71 | +[default7]: iteration 2864/ 3814 | consumed samples: 1466368 | consumed tokens: 3003121664 | elapsed time per iteration (s): 5.38 | learning rate: 2.577E-04 | global batch size: 512 | lm loss: 3.104965E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.082 | TFLOPs: 71.82 | +[default7]: iteration 2865/ 3814 | consumed samples: 1466880 | consumed tokens: 3004170240 | elapsed time per iteration (s): 5.35 | learning rate: 2.574E-04 | global batch size: 512 | lm loss: 3.142828E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.778 | TFLOPs: 72.34 | +[default7]: iteration 2866/ 3814 | consumed samples: 1467392 | consumed tokens: 3005218816 | elapsed time per iteration (s): 5.37 | learning rate: 2.571E-04 | global batch size: 512 | lm loss: 3.107908E+00 | loss scale: 131072.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.384 | TFLOPs: 72.04 | +[default7]: iteration 2867/ 3814 | consumed samples: 1467904 | consumed tokens: 3006267392 | elapsed time per iteration (s): 5.34 | learning rate: 2.568E-04 | global batch size: 512 | lm loss: 3.145082E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.901 | TFLOPs: 72.44 | +[default7]: iteration 2868/ 3814 | consumed samples: 1468416 | consumed tokens: 3007315968 | elapsed time per iteration (s): 5.33 | learning rate: 2.565E-04 | global batch size: 512 | lm loss: 3.111760E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.995 | TFLOPs: 72.51 | +[default7]: iteration 2869/ 3814 | consumed samples: 1468928 | consumed tokens: 3008364544 | elapsed time per iteration (s): 5.33 | learning rate: 2.562E-04 | global batch size: 512 | lm loss: 3.109988E+00 | loss scale: 131072.0 | grad norm: 0.128 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.068 | TFLOPs: 72.56 | +[default7]: iteration 2870/ 3814 | consumed samples: 1469440 | consumed tokens: 3009413120 | elapsed time per iteration (s): 5.37 | learning rate: 2.559E-04 | global batch size: 512 | lm loss: 3.101764E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.280 | TFLOPs: 71.97 | +[default7]: iteration 2871/ 3814 | consumed samples: 1469952 | consumed tokens: 3010461696 | elapsed time per iteration (s): 5.38 | learning rate: 2.556E-04 | global batch size: 512 | lm loss: 3.112419E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.150 | TFLOPs: 71.87 | +[default7]: iteration 2872/ 3814 | consumed samples: 1470464 | consumed tokens: 3011510272 | elapsed time per iteration (s): 5.36 | learning rate: 2.553E-04 | global batch size: 512 | lm loss: 3.125010E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.575 | TFLOPs: 72.19 | +[default7]: iteration 2873/ 3814 | consumed samples: 1470976 | consumed tokens: 3012558848 | elapsed time per iteration (s): 5.36 | learning rate: 2.550E-04 | global batch size: 512 | lm loss: 3.140187E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.490 | TFLOPs: 72.12 | +[default7]: iteration 2874/ 3814 | consumed samples: 1471488 | consumed tokens: 3013607424 | elapsed time per iteration (s): 5.37 | learning rate: 2.547E-04 | global batch size: 512 | lm loss: 3.137310E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.279 | TFLOPs: 71.97 | +[default7]: iteration 2875/ 3814 | consumed samples: 1472000 | consumed tokens: 3014656000 | elapsed time per iteration (s): 5.34 | learning rate: 2.543E-04 | global batch size: 512 | lm loss: 3.119904E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.791 | TFLOPs: 72.35 | +[default7]: iteration 2876/ 3814 | consumed samples: 1472512 | consumed tokens: 3015704576 | elapsed time per iteration (s): 5.41 | learning rate: 2.540E-04 | global batch size: 512 | lm loss: 3.126867E+00 | loss scale: 131072.0 | grad norm: 0.102 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.605 | TFLOPs: 71.46 | +[default7]: iteration 2877/ 3814 | consumed samples: 1473024 | consumed tokens: 3016753152 | elapsed time per iteration (s): 5.38 | learning rate: 2.537E-04 | global batch size: 512 | lm loss: 3.126165E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.199 | TFLOPs: 71.90 | +[default7]: iteration 2878/ 3814 | consumed samples: 1473536 | consumed tokens: 3017801728 | elapsed time per iteration (s): 5.39 | learning rate: 2.534E-04 | global batch size: 512 | lm loss: 3.120831E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.904 | TFLOPs: 71.68 | +[default7]: iteration 2879/ 3814 | consumed samples: 1474048 | consumed tokens: 3018850304 | elapsed time per iteration (s): 5.38 | learning rate: 2.531E-04 | global batch size: 512 | lm loss: 3.093262E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.253 | TFLOPs: 71.95 | +[default7]: iteration 2880/ 3814 | consumed samples: 1474560 | consumed tokens: 3019898880 | elapsed time per iteration (s): 5.38 | learning rate: 2.528E-04 | global batch size: 512 | lm loss: 3.128474E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.152 | TFLOPs: 71.87 | +[default7]: iteration 2881/ 3814 | consumed samples: 1475072 | consumed tokens: 3020947456 | elapsed time per iteration (s): 5.39 | learning rate: 2.525E-04 | global batch size: 512 | lm loss: 3.125416E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.972 | TFLOPs: 71.73 | +[default7]: iteration 2882/ 3814 | consumed samples: 1475584 | consumed tokens: 3021996032 | elapsed time per iteration (s): 5.37 | learning rate: 2.522E-04 | global batch size: 512 | lm loss: 3.114179E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.326 | TFLOPs: 72.00 | +[default7]: iteration 2883/ 3814 | consumed samples: 1476096 | consumed tokens: 3023044608 | elapsed time per iteration (s): 5.39 | learning rate: 2.519E-04 | global batch size: 512 | lm loss: 3.113755E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.905 | TFLOPs: 71.68 | +[default7]: iteration 2884/ 3814 | consumed samples: 1476608 | consumed tokens: 3024093184 | elapsed time per iteration (s): 5.39 | learning rate: 2.516E-04 | global batch size: 512 | lm loss: 3.124827E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.977 | TFLOPs: 71.74 | +[default7]: iteration 2885/ 3814 | consumed samples: 1477120 | consumed tokens: 3025141760 | elapsed time per iteration (s): 5.40 | learning rate: 2.513E-04 | global batch size: 512 | lm loss: 3.126220E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.730 | TFLOPs: 71.55 | +[default7]: iteration 2886/ 3814 | consumed samples: 1477632 | consumed tokens: 3026190336 | elapsed time per iteration (s): 5.39 | learning rate: 2.510E-04 | global batch size: 512 | lm loss: 3.093798E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.988 | TFLOPs: 71.75 | +[default7]: iteration 2887/ 3814 | consumed samples: 1478144 | consumed tokens: 3027238912 | elapsed time per iteration (s): 5.39 | learning rate: 2.507E-04 | global batch size: 512 | lm loss: 3.137160E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.920 | TFLOPs: 71.69 | +[default7]: iteration 2888/ 3814 | consumed samples: 1478656 | consumed tokens: 3028287488 | elapsed time per iteration (s): 5.39 | learning rate: 2.504E-04 | global batch size: 512 | lm loss: 3.119789E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.075 | TFLOPs: 71.81 | +[default7]: iteration 2889/ 3814 | consumed samples: 1479168 | consumed tokens: 3029336064 | elapsed time per iteration (s): 5.40 | learning rate: 2.501E-04 | global batch size: 512 | lm loss: 3.102339E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.882 | TFLOPs: 71.67 | +[default7]: iteration 2890/ 3814 | consumed samples: 1479680 | consumed tokens: 3030384640 | elapsed time per iteration (s): 5.40 | learning rate: 2.497E-04 | global batch size: 512 | lm loss: 3.116312E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.809 | TFLOPs: 71.61 | +[default7]: iteration 2891/ 3814 | consumed samples: 1480192 | consumed tokens: 3031433216 | elapsed time per iteration (s): 5.37 | learning rate: 2.494E-04 | global batch size: 512 | lm loss: 3.098297E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.415 | TFLOPs: 72.07 | +[default7]: iteration 2892/ 3814 | consumed samples: 1480704 | consumed tokens: 3032481792 | elapsed time per iteration (s): 5.39 | learning rate: 2.491E-04 | global batch size: 512 | lm loss: 3.119901E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.961 | TFLOPs: 71.73 | +[default7]: iteration 2893/ 3814 | consumed samples: 1481216 | consumed tokens: 3033530368 | elapsed time per iteration (s): 5.40 | learning rate: 2.488E-04 | global batch size: 512 | lm loss: 3.081699E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.875 | TFLOPs: 71.66 | +[default7]: iteration 2894/ 3814 | consumed samples: 1481728 | consumed tokens: 3034578944 | elapsed time per iteration (s): 5.42 | learning rate: 2.485E-04 | global batch size: 512 | lm loss: 3.158179E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.511 | TFLOPs: 71.39 | +[default7]: iteration 2895/ 3814 | consumed samples: 1482240 | consumed tokens: 3035627520 | elapsed time per iteration (s): 5.38 | learning rate: 2.482E-04 | global batch size: 512 | lm loss: 3.125536E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.129 | TFLOPs: 71.85 | +[default7]: iteration 2896/ 3814 | consumed samples: 1482752 | consumed tokens: 3036676096 | elapsed time per iteration (s): 5.39 | learning rate: 2.479E-04 | global batch size: 512 | lm loss: 3.125106E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.940 | TFLOPs: 71.71 | +[default7]: iteration 2897/ 3814 | consumed samples: 1483264 | consumed tokens: 3037724672 | elapsed time per iteration (s): 5.38 | learning rate: 2.476E-04 | global batch size: 512 | lm loss: 3.141814E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.192 | TFLOPs: 71.90 | +[default7]: iteration 2898/ 3814 | consumed samples: 1483776 | consumed tokens: 3038773248 | elapsed time per iteration (s): 5.39 | learning rate: 2.473E-04 | global batch size: 512 | lm loss: 3.089550E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.016 | TFLOPs: 71.77 | +[default7]: iteration 2899/ 3814 | consumed samples: 1484288 | consumed tokens: 3039821824 | elapsed time per iteration (s): 5.38 | learning rate: 2.470E-04 | global batch size: 512 | lm loss: 3.126028E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.178 | TFLOPs: 71.89 | +[default7]: iteration 2900/ 3814 | consumed samples: 1484800 | consumed tokens: 3040870400 | elapsed time per iteration (s): 5.40 | learning rate: 2.467E-04 | global batch size: 512 | lm loss: 3.109234E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.835 | TFLOPs: 71.63 | +[default7]: iteration 2901/ 3814 | consumed samples: 1485312 | consumed tokens: 3041918976 | elapsed time per iteration (s): 5.39 | learning rate: 2.464E-04 | global batch size: 512 | lm loss: 3.130383E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.911 | TFLOPs: 71.69 | +[default7]: iteration 2902/ 3814 | consumed samples: 1485824 | consumed tokens: 3042967552 | elapsed time per iteration (s): 5.39 | learning rate: 2.461E-04 | global batch size: 512 | lm loss: 3.112978E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.031 | TFLOPs: 71.78 | +[default7]: iteration 2903/ 3814 | consumed samples: 1486336 | consumed tokens: 3044016128 | elapsed time per iteration (s): 5.37 | learning rate: 2.458E-04 | global batch size: 512 | lm loss: 3.095571E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.354 | TFLOPs: 72.02 | +[default7]: iteration 2904/ 3814 | consumed samples: 1486848 | consumed tokens: 3045064704 | elapsed time per iteration (s): 5.38 | learning rate: 2.455E-04 | global batch size: 512 | lm loss: 3.095084E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.086 | TFLOPs: 71.82 | +[default7]: iteration 2905/ 3814 | consumed samples: 1487360 | consumed tokens: 3046113280 | elapsed time per iteration (s): 5.41 | learning rate: 2.452E-04 | global batch size: 512 | lm loss: 3.123356E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.674 | TFLOPs: 71.51 | +[default7]: iteration 2906/ 3814 | consumed samples: 1487872 | consumed tokens: 3047161856 | elapsed time per iteration (s): 5.42 | learning rate: 2.449E-04 | global batch size: 512 | lm loss: 3.098557E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.473 | TFLOPs: 71.36 | +[default7]: iteration 2907/ 3814 | consumed samples: 1488384 | consumed tokens: 3048210432 | elapsed time per iteration (s): 5.39 | learning rate: 2.446E-04 | global batch size: 512 | lm loss: 3.136846E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.912 | TFLOPs: 71.69 | +[default7]: iteration 2908/ 3814 | consumed samples: 1488896 | consumed tokens: 3049259008 | elapsed time per iteration (s): 5.40 | learning rate: 2.443E-04 | global batch size: 512 | lm loss: 3.129023E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.762 | TFLOPs: 71.57 | +[default7]: iteration 2909/ 3814 | consumed samples: 1489408 | consumed tokens: 3050307584 | elapsed time per iteration (s): 5.40 | learning rate: 2.440E-04 | global batch size: 512 | lm loss: 3.121024E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.787 | TFLOPs: 71.59 | +[default7]: iteration 2910/ 3814 | consumed samples: 1489920 | consumed tokens: 3051356160 | elapsed time per iteration (s): 5.38 | learning rate: 2.437E-04 | global batch size: 512 | lm loss: 3.110401E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.189 | TFLOPs: 71.90 | +[default7]: iteration 2911/ 3814 | consumed samples: 1490432 | consumed tokens: 3052404736 | elapsed time per iteration (s): 5.44 | learning rate: 2.434E-04 | global batch size: 512 | lm loss: 3.103819E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.189 | TFLOPs: 71.14 | +[default7]: iteration 2912/ 3814 | consumed samples: 1490944 | consumed tokens: 3053453312 | elapsed time per iteration (s): 5.41 | learning rate: 2.431E-04 | global batch size: 512 | lm loss: 3.115181E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.630 | TFLOPs: 71.47 | +[default7]: iteration 2913/ 3814 | consumed samples: 1491456 | consumed tokens: 3054501888 | elapsed time per iteration (s): 5.40 | learning rate: 2.428E-04 | global batch size: 512 | lm loss: 3.118557E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 2914/ 3814 | consumed samples: 1491968 | consumed tokens: 3055550464 | elapsed time per iteration (s): 5.37 | learning rate: 2.425E-04 | global batch size: 512 | lm loss: 3.094736E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.279 | TFLOPs: 71.97 | +[default7]: iteration 2915/ 3814 | consumed samples: 1492480 | consumed tokens: 3056599040 | elapsed time per iteration (s): 5.34 | learning rate: 2.422E-04 | global batch size: 512 | lm loss: 3.093688E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.816 | TFLOPs: 72.37 | +[default7]: iteration 2916/ 3814 | consumed samples: 1492992 | consumed tokens: 3057647616 | elapsed time per iteration (s): 5.39 | learning rate: 2.419E-04 | global batch size: 512 | lm loss: 3.097134E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.033 | TFLOPs: 71.78 | +[default7]: iteration 2917/ 3814 | consumed samples: 1493504 | consumed tokens: 3058696192 | elapsed time per iteration (s): 5.36 | learning rate: 2.416E-04 | global batch size: 512 | lm loss: 3.124445E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.471 | TFLOPs: 72.11 | +[default7]: iteration 2918/ 3814 | consumed samples: 1494016 | consumed tokens: 3059744768 | elapsed time per iteration (s): 5.36 | learning rate: 2.413E-04 | global batch size: 512 | lm loss: 3.098928E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.504 | TFLOPs: 72.14 | +[default7]: iteration 2919/ 3814 | consumed samples: 1494528 | consumed tokens: 3060793344 | elapsed time per iteration (s): 5.37 | learning rate: 2.410E-04 | global batch size: 512 | lm loss: 3.125724E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.405 | TFLOPs: 72.06 | +[default7]: iteration 2920/ 3814 | consumed samples: 1495040 | consumed tokens: 3061841920 | elapsed time per iteration (s): 5.33 | learning rate: 2.407E-04 | global batch size: 512 | lm loss: 3.096027E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.069 | TFLOPs: 72.56 | +[default7]: iteration 2921/ 3814 | consumed samples: 1495552 | consumed tokens: 3062890496 | elapsed time per iteration (s): 5.41 | learning rate: 2.404E-04 | global batch size: 512 | lm loss: 3.128818E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.709 | TFLOPs: 71.54 | +[default7]: iteration 2922/ 3814 | consumed samples: 1496064 | consumed tokens: 3063939072 | elapsed time per iteration (s): 5.39 | learning rate: 2.401E-04 | global batch size: 512 | lm loss: 3.116856E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.966 | TFLOPs: 71.73 | +[default7]: iteration 2923/ 3814 | consumed samples: 1496576 | consumed tokens: 3064987648 | elapsed time per iteration (s): 5.37 | learning rate: 2.398E-04 | global batch size: 512 | lm loss: 3.125180E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.423 | TFLOPs: 72.07 | +[default7]: iteration 2924/ 3814 | consumed samples: 1497088 | consumed tokens: 3066036224 | elapsed time per iteration (s): 5.37 | learning rate: 2.395E-04 | global batch size: 512 | lm loss: 3.118690E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.298 | TFLOPs: 71.98 | +[default7]: iteration 2925/ 3814 | consumed samples: 1497600 | consumed tokens: 3067084800 | elapsed time per iteration (s): 5.41 | learning rate: 2.392E-04 | global batch size: 512 | lm loss: 3.101405E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.699 | TFLOPs: 71.53 | +[default7]: iteration 2926/ 3814 | consumed samples: 1498112 | consumed tokens: 3068133376 | elapsed time per iteration (s): 5.39 | learning rate: 2.389E-04 | global batch size: 512 | lm loss: 3.085418E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.974 | TFLOPs: 71.73 | +[default7]: iteration 2927/ 3814 | consumed samples: 1498624 | consumed tokens: 3069181952 | elapsed time per iteration (s): 5.37 | learning rate: 2.386E-04 | global batch size: 512 | lm loss: 3.124998E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.302 | TFLOPs: 71.98 | +[default7]: iteration 2928/ 3814 | consumed samples: 1499136 | consumed tokens: 3070230528 | elapsed time per iteration (s): 5.38 | learning rate: 2.384E-04 | global batch size: 512 | lm loss: 3.084664E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.253 | TFLOPs: 71.95 | +[default7]: iteration 2929/ 3814 | consumed samples: 1499648 | consumed tokens: 3071279104 | elapsed time per iteration (s): 5.35 | learning rate: 2.381E-04 | global batch size: 512 | lm loss: 3.116598E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.655 | TFLOPs: 72.25 | +[default7]: iteration 2930/ 3814 | consumed samples: 1500160 | consumed tokens: 3072327680 | elapsed time per iteration (s): 5.36 | learning rate: 2.378E-04 | global batch size: 512 | lm loss: 3.098368E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.523 | TFLOPs: 72.15 | +[default7]: iteration 2931/ 3814 | consumed samples: 1500672 | consumed tokens: 3073376256 | elapsed time per iteration (s): 5.38 | learning rate: 2.375E-04 | global batch size: 512 | lm loss: 3.086167E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.163 | TFLOPs: 71.88 | +[default7]: iteration 2932/ 3814 | consumed samples: 1501184 | consumed tokens: 3074424832 | elapsed time per iteration (s): 5.38 | learning rate: 2.372E-04 | global batch size: 512 | lm loss: 3.138321E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.254 | TFLOPs: 71.95 | +[default7]: iteration 2933/ 3814 | consumed samples: 1501696 | consumed tokens: 3075473408 | elapsed time per iteration (s): 5.37 | learning rate: 2.369E-04 | global batch size: 512 | lm loss: 3.086530E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.371 | TFLOPs: 72.03 | +[default7]: iteration 2934/ 3814 | consumed samples: 1502208 | consumed tokens: 3076521984 | elapsed time per iteration (s): 5.40 | learning rate: 2.366E-04 | global batch size: 512 | lm loss: 3.097593E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.766 | TFLOPs: 71.58 | +[default7]: iteration 2935/ 3814 | consumed samples: 1502720 | consumed tokens: 3077570560 | elapsed time per iteration (s): 5.37 | learning rate: 2.363E-04 | global batch size: 512 | lm loss: 3.131560E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.261 | TFLOPs: 71.95 | +[default7]: iteration 2936/ 3814 | consumed samples: 1503232 | consumed tokens: 3078619136 | elapsed time per iteration (s): 5.38 | learning rate: 2.360E-04 | global batch size: 512 | lm loss: 3.089220E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.229 | TFLOPs: 71.93 | +[default7]: iteration 2937/ 3814 | consumed samples: 1503744 | consumed tokens: 3079667712 | elapsed time per iteration (s): 5.38 | learning rate: 2.357E-04 | global batch size: 512 | lm loss: 3.073061E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.160 | TFLOPs: 71.88 | +[default7]: iteration 2938/ 3814 | consumed samples: 1504256 | consumed tokens: 3080716288 | elapsed time per iteration (s): 5.39 | learning rate: 2.354E-04 | global batch size: 512 | lm loss: 3.122834E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.055 | TFLOPs: 71.80 | +[default7]: iteration 2939/ 3814 | consumed samples: 1504768 | consumed tokens: 3081764864 | elapsed time per iteration (s): 5.39 | learning rate: 2.351E-04 | global batch size: 512 | lm loss: 3.127548E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.930 | TFLOPs: 71.70 | +[default7]: iteration 2940/ 3814 | consumed samples: 1505280 | consumed tokens: 3082813440 | elapsed time per iteration (s): 5.40 | learning rate: 2.348E-04 | global batch size: 512 | lm loss: 3.119563E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.756 | TFLOPs: 71.57 | +[default7]: iteration 2941/ 3814 | consumed samples: 1505792 | consumed tokens: 3083862016 | elapsed time per iteration (s): 5.39 | learning rate: 2.345E-04 | global batch size: 512 | lm loss: 3.081630E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.972 | TFLOPs: 71.73 | +[default7]: iteration 2942/ 3814 | consumed samples: 1506304 | consumed tokens: 3084910592 | elapsed time per iteration (s): 5.40 | learning rate: 2.342E-04 | global batch size: 512 | lm loss: 3.073965E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.846 | TFLOPs: 71.64 | +[default7]: iteration 2943/ 3814 | consumed samples: 1506816 | consumed tokens: 3085959168 | elapsed time per iteration (s): 5.40 | learning rate: 2.340E-04 | global batch size: 512 | lm loss: 3.132007E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.820 | TFLOPs: 71.62 | +[default7]: iteration 2944/ 3814 | consumed samples: 1507328 | consumed tokens: 3087007744 | elapsed time per iteration (s): 5.40 | learning rate: 2.337E-04 | global batch size: 512 | lm loss: 3.075297E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.757 | TFLOPs: 71.57 | +[default7]: iteration 2945/ 3814 | consumed samples: 1507840 | consumed tokens: 3088056320 | elapsed time per iteration (s): 5.40 | learning rate: 2.334E-04 | global batch size: 512 | lm loss: 3.103653E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.815 | TFLOPs: 71.62 | +[default7]: iteration 2946/ 3814 | consumed samples: 1508352 | consumed tokens: 3089104896 | elapsed time per iteration (s): 5.45 | learning rate: 2.331E-04 | global batch size: 512 | lm loss: 3.089411E+00 | loss scale: 131072.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.021 | TFLOPs: 71.02 | +[default7]: iteration 2947/ 3814 | consumed samples: 1508864 | consumed tokens: 3090153472 | elapsed time per iteration (s): 5.39 | learning rate: 2.328E-04 | global batch size: 512 | lm loss: 3.101559E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.067 | TFLOPs: 71.80 | +[default7]: iteration 2948/ 3814 | consumed samples: 1509376 | consumed tokens: 3091202048 | elapsed time per iteration (s): 5.40 | learning rate: 2.325E-04 | global batch size: 512 | lm loss: 3.137365E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 2949/ 3814 | consumed samples: 1509888 | consumed tokens: 3092250624 | elapsed time per iteration (s): 5.41 | learning rate: 2.322E-04 | global batch size: 512 | lm loss: 3.092537E+00 | loss scale: 131072.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.610 | TFLOPs: 71.46 | +[default7]: iteration 2950/ 3814 | consumed samples: 1510400 | consumed tokens: 3093299200 | elapsed time per iteration (s): 5.40 | learning rate: 2.319E-04 | global batch size: 512 | lm loss: 3.114858E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.856 | TFLOPs: 71.65 | +[default7]: iteration 2951/ 3814 | consumed samples: 1510912 | consumed tokens: 3094347776 | elapsed time per iteration (s): 5.43 | learning rate: 2.316E-04 | global batch size: 512 | lm loss: 3.113443E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.269 | TFLOPs: 71.20 | +[default7]: iteration 2952/ 3814 | consumed samples: 1511424 | consumed tokens: 3095396352 | elapsed time per iteration (s): 5.39 | learning rate: 2.313E-04 | global batch size: 512 | lm loss: 3.119804E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.977 | TFLOPs: 71.74 | +[default7]: iteration 2953/ 3814 | consumed samples: 1511936 | consumed tokens: 3096444928 | elapsed time per iteration (s): 5.40 | learning rate: 2.311E-04 | global batch size: 512 | lm loss: 3.147708E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.864 | TFLOPs: 71.65 | +[default7]: iteration 2954/ 3814 | consumed samples: 1512448 | consumed tokens: 3097493504 | elapsed time per iteration (s): 5.39 | learning rate: 2.308E-04 | global batch size: 512 | lm loss: 3.102846E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.035 | TFLOPs: 71.78 | +[default7]: iteration 2955/ 3814 | consumed samples: 1512960 | consumed tokens: 3098542080 | elapsed time per iteration (s): 5.42 | learning rate: 2.305E-04 | global batch size: 512 | lm loss: 3.130436E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.490 | TFLOPs: 71.37 | +[default7]: iteration 2956/ 3814 | consumed samples: 1513472 | consumed tokens: 3099590656 | elapsed time per iteration (s): 5.39 | learning rate: 2.302E-04 | global batch size: 512 | lm loss: 3.095622E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.005 | TFLOPs: 71.76 | +[default7]: iteration 2957/ 3814 | consumed samples: 1513984 | consumed tokens: 3100639232 | elapsed time per iteration (s): 5.37 | learning rate: 2.299E-04 | global batch size: 512 | lm loss: 3.082659E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.285 | TFLOPs: 71.97 | +[default7]: iteration 2958/ 3814 | consumed samples: 1514496 | consumed tokens: 3101687808 | elapsed time per iteration (s): 5.39 | learning rate: 2.296E-04 | global batch size: 512 | lm loss: 3.092309E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.017 | TFLOPs: 71.77 | +[default7]: iteration 2959/ 3814 | consumed samples: 1515008 | consumed tokens: 3102736384 | elapsed time per iteration (s): 5.37 | learning rate: 2.293E-04 | global batch size: 512 | lm loss: 3.106500E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.383 | TFLOPs: 72.04 | +[default7]: iteration 2960/ 3814 | consumed samples: 1515520 | consumed tokens: 3103784960 | elapsed time per iteration (s): 5.37 | learning rate: 2.290E-04 | global batch size: 512 | lm loss: 3.116173E+00 | loss scale: 131072.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.303 | TFLOPs: 71.98 | +[default7]: iteration 2961/ 3814 | consumed samples: 1516032 | consumed tokens: 3104833536 | elapsed time per iteration (s): 5.39 | learning rate: 2.288E-04 | global batch size: 512 | lm loss: 3.063500E+00 | loss scale: 131072.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.000 | TFLOPs: 71.75 | +[default7]: iteration 2962/ 3814 | consumed samples: 1516544 | consumed tokens: 3105882112 | elapsed time per iteration (s): 5.39 | learning rate: 2.285E-04 | global batch size: 512 | lm loss: 3.106681E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.061 | TFLOPs: 71.80 | +[default7]: iteration 2963/ 3814 | consumed samples: 1517056 | consumed tokens: 3106930688 | elapsed time per iteration (s): 5.40 | learning rate: 2.282E-04 | global batch size: 512 | lm loss: 3.107052E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.737 | TFLOPs: 71.56 | +[default7]: iteration 2964/ 3814 | consumed samples: 1517568 | consumed tokens: 3107979264 | elapsed time per iteration (s): 5.39 | learning rate: 2.279E-04 | global batch size: 512 | lm loss: 3.127332E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.962 | TFLOPs: 71.73 | +[default7]: iteration 2965/ 3814 | consumed samples: 1518080 | consumed tokens: 3109027840 | elapsed time per iteration (s): 5.40 | learning rate: 2.276E-04 | global batch size: 512 | lm loss: 3.103745E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.888 | TFLOPs: 71.67 | +[default7]: iteration 2966/ 3814 | consumed samples: 1518592 | consumed tokens: 3110076416 | elapsed time per iteration (s): 5.37 | learning rate: 2.273E-04 | global batch size: 512 | lm loss: 3.130680E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.343 | TFLOPs: 72.01 | +[default7]: iteration 2967/ 3814 | consumed samples: 1519104 | consumed tokens: 3111124992 | elapsed time per iteration (s): 5.38 | learning rate: 2.270E-04 | global batch size: 512 | lm loss: 3.115657E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.252 | TFLOPs: 71.94 | +[default7]: iteration 2968/ 3814 | consumed samples: 1519616 | consumed tokens: 3112173568 | elapsed time per iteration (s): 5.38 | learning rate: 2.268E-04 | global batch size: 512 | lm loss: 3.114185E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.246 | TFLOPs: 71.94 | +[default7]: iteration 2969/ 3814 | consumed samples: 1520128 | consumed tokens: 3113222144 | elapsed time per iteration (s): 5.41 | learning rate: 2.265E-04 | global batch size: 512 | lm loss: 3.108212E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.623 | TFLOPs: 71.47 | +[default7]: iteration 2970/ 3814 | consumed samples: 1520640 | consumed tokens: 3114270720 | elapsed time per iteration (s): 5.38 | learning rate: 2.262E-04 | global batch size: 512 | lm loss: 3.079112E+00 | loss scale: 131072.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.102 | TFLOPs: 71.83 | +[default7]: iteration 2971/ 3814 | consumed samples: 1521152 | consumed tokens: 3115319296 | elapsed time per iteration (s): 5.35 | learning rate: 2.259E-04 | global batch size: 512 | lm loss: 3.110375E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.715 | TFLOPs: 72.29 | +[default7]: iteration 2972/ 3814 | consumed samples: 1521664 | consumed tokens: 3116367872 | elapsed time per iteration (s): 5.34 | learning rate: 2.256E-04 | global batch size: 512 | lm loss: 3.093405E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.853 | TFLOPs: 72.40 | +[default7]: iteration 2973/ 3814 | consumed samples: 1522176 | consumed tokens: 3117416448 | elapsed time per iteration (s): 5.37 | learning rate: 2.253E-04 | global batch size: 512 | lm loss: 3.091674E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.343 | TFLOPs: 72.01 | +[default7]: iteration 2974/ 3814 | consumed samples: 1522688 | consumed tokens: 3118465024 | elapsed time per iteration (s): 5.37 | learning rate: 2.251E-04 | global batch size: 512 | lm loss: 3.101385E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.325 | TFLOPs: 72.00 | +[default7]: iteration 2975/ 3814 | consumed samples: 1523200 | consumed tokens: 3119513600 | elapsed time per iteration (s): 5.38 | learning rate: 2.248E-04 | global batch size: 512 | lm loss: 3.094443E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.212 | TFLOPs: 71.91 | +[default7]: iteration 2976/ 3814 | consumed samples: 1523712 | consumed tokens: 3120562176 | elapsed time per iteration (s): 5.36 | learning rate: 2.245E-04 | global batch size: 512 | lm loss: 3.121849E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.575 | TFLOPs: 72.19 | +[default7]: iteration 2977/ 3814 | consumed samples: 1524224 | consumed tokens: 3121610752 | elapsed time per iteration (s): 5.39 | learning rate: 2.242E-04 | global batch size: 512 | lm loss: 3.085297E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.959 | TFLOPs: 71.72 | +[default7]: iteration 2978/ 3814 | consumed samples: 1524736 | consumed tokens: 3122659328 | elapsed time per iteration (s): 5.39 | learning rate: 2.239E-04 | global batch size: 512 | lm loss: 3.067587E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.944 | TFLOPs: 71.71 | +[default7]: iteration 2979/ 3814 | consumed samples: 1525248 | consumed tokens: 3123707904 | elapsed time per iteration (s): 5.38 | learning rate: 2.236E-04 | global batch size: 512 | lm loss: 3.130737E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.198 | TFLOPs: 71.90 | +[default7]: iteration 2980/ 3814 | consumed samples: 1525760 | consumed tokens: 3124756480 | elapsed time per iteration (s): 5.40 | learning rate: 2.234E-04 | global batch size: 512 | lm loss: 3.109448E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.734 | TFLOPs: 71.55 | +[default7]: iteration 2981/ 3814 | consumed samples: 1526272 | consumed tokens: 3125805056 | elapsed time per iteration (s): 5.40 | learning rate: 2.231E-04 | global batch size: 512 | lm loss: 3.123519E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.803 | TFLOPs: 71.61 | +[default7]: iteration 2982/ 3814 | consumed samples: 1526784 | consumed tokens: 3126853632 | elapsed time per iteration (s): 5.38 | learning rate: 2.228E-04 | global batch size: 512 | lm loss: 3.091935E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.163 | TFLOPs: 71.88 | +[default7]: iteration 2983/ 3814 | consumed samples: 1527296 | consumed tokens: 3127902208 | elapsed time per iteration (s): 5.38 | learning rate: 2.225E-04 | global batch size: 512 | lm loss: 3.147755E+00 | loss scale: 131072.0 | grad norm: 0.104 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.194 | TFLOPs: 71.90 | +[default7]: iteration 2984/ 3814 | consumed samples: 1527808 | consumed tokens: 3128950784 | elapsed time per iteration (s): 5.38 | learning rate: 2.222E-04 | global batch size: 512 | lm loss: 3.083830E+00 | loss scale: 131072.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.206 | TFLOPs: 71.91 | +[default7]: iteration 2985/ 3814 | consumed samples: 1528320 | consumed tokens: 3129999360 | elapsed time per iteration (s): 5.40 | learning rate: 2.220E-04 | global batch size: 512 | lm loss: 3.100036E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.836 | TFLOPs: 71.63 | +[default7]: iteration 2986/ 3814 | consumed samples: 1528832 | consumed tokens: 3131047936 | elapsed time per iteration (s): 5.39 | learning rate: 2.217E-04 | global batch size: 512 | lm loss: 3.124122E+00 | loss scale: 131072.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.985 | TFLOPs: 71.74 | +[default7]: iteration 2987/ 3814 | consumed samples: 1529344 | consumed tokens: 3132096512 | elapsed time per iteration (s): 5.40 | learning rate: 2.214E-04 | global batch size: 512 | lm loss: 3.098118E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.861 | TFLOPs: 71.65 | +[default7]: iteration 2988/ 3814 | consumed samples: 1529856 | consumed tokens: 3133145088 | elapsed time per iteration (s): 5.39 | learning rate: 2.211E-04 | global batch size: 512 | lm loss: 3.069372E+00 | loss scale: 131072.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.945 | TFLOPs: 71.71 | +[default7]: iteration 2989/ 3814 | consumed samples: 1530368 | consumed tokens: 3134193664 | elapsed time per iteration (s): 5.40 | learning rate: 2.208E-04 | global batch size: 512 | lm loss: 3.114778E+00 | loss scale: 131072.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.891 | TFLOPs: 71.67 | +[default7]: iteration 2990/ 3814 | consumed samples: 1530880 | consumed tokens: 3135242240 | elapsed time per iteration (s): 5.41 | learning rate: 2.206E-04 | global batch size: 512 | lm loss: 3.117028E+00 | loss scale: 131072.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.677 | TFLOPs: 71.51 | +[default7]: iteration 2991/ 3814 | consumed samples: 1531392 | consumed tokens: 3136290816 | elapsed time per iteration (s): 5.38 | learning rate: 2.203E-04 | global batch size: 512 | lm loss: 3.101748E+00 | loss scale: 131072.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.114 | TFLOPs: 71.84 | +[default7]: iteration 2992/ 3814 | consumed samples: 1531904 | consumed tokens: 3137339392 | elapsed time per iteration (s): 5.38 | learning rate: 2.200E-04 | global batch size: 512 | lm loss: 3.107084E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.147 | TFLOPs: 71.87 | +[default7]: iteration 2993/ 3814 | consumed samples: 1532416 | consumed tokens: 3138387968 | elapsed time per iteration (s): 5.40 | learning rate: 2.197E-04 | global batch size: 512 | lm loss: 3.076207E+00 | loss scale: 131072.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.824 | TFLOPs: 71.62 | +[default7]: iteration 2994/ 3814 | consumed samples: 1532928 | consumed tokens: 3139436544 | elapsed time per iteration (s): 5.39 | learning rate: 2.195E-04 | global batch size: 512 | lm loss: 3.113673E+00 | loss scale: 131072.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.033 | TFLOPs: 71.78 | +[default7]: iteration 2995/ 3814 | consumed samples: 1533440 | consumed tokens: 3140485120 | elapsed time per iteration (s): 5.40 | learning rate: 2.192E-04 | global batch size: 512 | lm loss: 3.087347E+00 | loss scale: 131072.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.825 | TFLOPs: 71.62 | +[default7]: iteration 2996/ 3814 | consumed samples: 1533952 | consumed tokens: 3141533696 | elapsed time per iteration (s): 5.41 | learning rate: 2.189E-04 | global batch size: 512 | lm loss: 3.091863E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.719 | TFLOPs: 71.54 | +[default7]: iteration 2997/ 3814 | consumed samples: 1534464 | consumed tokens: 3142582272 | elapsed time per iteration (s): 5.41 | learning rate: 2.186E-04 | global batch size: 512 | lm loss: 3.055050E+00 | loss scale: 131072.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.674 | TFLOPs: 71.51 | +[default7]: iteration 2998/ 3814 | consumed samples: 1534976 | consumed tokens: 3143630848 | elapsed time per iteration (s): 5.40 | learning rate: 2.183E-04 | global batch size: 512 | lm loss: 3.076986E+00 | loss scale: 131072.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.869 | TFLOPs: 71.66 | +[default7]: iteration 2999/ 3814 | consumed samples: 1535488 | consumed tokens: 3144679424 | elapsed time per iteration (s): 5.39 | learning rate: 2.181E-04 | global batch size: 512 | lm loss: 3.108120E+00 | loss scale: 131072.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.075 | TFLOPs: 71.81 | +[default7]: iteration 3000/ 3814 | consumed samples: 1536000 | consumed tokens: 3145728000 | elapsed time per iteration (s): 5.37 | learning rate: 2.178E-04 | global batch size: 512 | lm loss: 3.116926E+00 | loss scale: 262144.0 | grad norm: 0.055 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.327 | TFLOPs: 72.00 | +[default1]:[2023-02-16 10:50:53,990] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_01-model_01-model_states.pt... +[default7]:------------------------------------------------------------------------------------------ +[default7]:valid loss at iteration 3000 | lm loss value: 3.079629E+00 | lm loss PPL: 2.175034E+01 | +[default7]:------------------------------------------------------------------------------------------ +[default0]:saving checkpoint at iteration 3000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 10:50:53,987] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step3000 is begin to save! +[default0]:[2023-02-16 10:50:53,990] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_01-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,149] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,149] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_04-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,160] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,161] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_04-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:54,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,231] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,217] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_05-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:54,281] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,282] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_06-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,292] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_06-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,347] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,360] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,361] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_07-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:54,423] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,424] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_08-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,409] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,410] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:54,467] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,467] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,486] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,524] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,525] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_10-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:54,600] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,600] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_11-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,559] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,559] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_10-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:54,633] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,634] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_11-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,672] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,673] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,702] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,702] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,742] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,743] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_13-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,773] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,773] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:54,834] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,835] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,811] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,812] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_14-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:54,878] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,879] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_15-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,903] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,903] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_15-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:54,937] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,938] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_16-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:54,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:54,996] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_17-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:54,963] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:54,964] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_16-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,025] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,025] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_17-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,057] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,058] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,085] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,086] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_18-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,144] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,145] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_19-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,117] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,117] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:55,178] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,179] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,208] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,248] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,248] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_21-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,274] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,274] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_21-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,317] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_22-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:55,388] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,388] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_23-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,349] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,349] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_22-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,417] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,418] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_23-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,456] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,456] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,487] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,488] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,526] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,527] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_25-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:55,589] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,590] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_26-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,551] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,551] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_25-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,610] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,611] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_26-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,649] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,650] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,681] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,681] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_27-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,743] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,743] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_28-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,724] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,725] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_28-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:55,783] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,784] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,802] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,802] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,842] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,843] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_30-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:55,901] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,902] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:55,861] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:55,935] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:55,935] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:55,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:55,976] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_32-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:56,004] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,004] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_32-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:56,043] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:56,043] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:56,071] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,071] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_33-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:56,138] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,139] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 10:50:56,109] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:56,110] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:56,176] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:56,177] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_35-model_01-model_states.pt... +[default0]:[2023-02-16 10:50:56,201] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,201] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,240] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:56,240] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 10:50:56,241] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 10:50:56,242] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_01_model_states.pt +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,242] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 10:50:56,244] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:50:56,264] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,265] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_37-model_00-model_states.pt... +[default0]:[2023-02-16 10:50:56,265] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 10:50:56,267] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_00_model_states.pt +[default0]:[2023-02-16 10:50:56,267] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 10:50:56,269] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 10:50:56,293] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 10:50:56,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,809] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,809] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:56,784] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,784] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,784] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:56,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:56,817] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:56,817] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,882] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,882] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,882] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,837] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,837] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,837] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,919] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,919] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,952] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,952] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,952] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,914] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,914] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,914] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,938] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,939] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,939] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,947] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,948] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,971] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,972] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,972] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,918] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,918] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,918] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:56,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,926] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,926] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,926] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,970] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,970] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,970] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,981] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,981] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,981] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,946] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,946] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,946] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,924] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,924] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,924] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:56,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:56,933] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:56,933] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:56,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:56,992] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:56,992] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:56,982] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:56,982] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:56,983] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:56,933] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:56,933] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:56,934] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,966] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,966] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,941] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,941] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:56,931] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:56,931] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:56,931] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:56,917] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,917] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,918] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:56,944] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:56,944] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:56,944] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:56,980] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,980] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,980] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:56,928] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,929] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,929] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,948] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,948] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,948] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,955] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,930] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,930] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,930] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:56,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:56,997] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:56,997] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:57,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:57,002] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:57,002] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,966] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,966] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,966] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,975] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:56,994] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:56,994] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:56,994] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,975] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,975] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,941] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,942] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,942] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,947] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,947] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,947] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,973] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,974] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,974] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:56,956] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:56,956] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:57,046] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:57,046] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:57,046] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:56,954] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:56,954] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:56,954] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:56,969] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:56,969] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:56,969] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:56,997] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:56,997] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:56,997] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:57,024] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:57,024] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:57,024] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:56,978] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:56,978] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:56,978] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:57,001] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:57,001] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:57,002] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:57,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:57,018] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:57,018] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default5]:[2023-02-16 10:50:57,007] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:57,007] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:57,007] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default1]:[2023-02-16 10:50:57,041] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 10:50:57,041] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 10:50:57,041] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default0]:[2023-02-16 10:50:57,047] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 10:50:57,047] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 10:50:57,047] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default4]:[2023-02-16 10:50:57,002] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 10:50:57,003] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 10:50:57,003] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:57,012] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:57,012] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:57,012] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:57,013] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:57,013] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:57,013] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:57,037] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:57,038] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:57,038] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default3]:[2023-02-16 10:50:57,018] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 10:50:57,018] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 10:50:57,018] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default2]:[2023-02-16 10:50:57,064] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 10:50:57,064] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 10:50:57,065] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default6]:[2023-02-16 10:50:57,060] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 10:50:57,060] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 10:50:57,060] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:[2023-02-16 10:50:57,091] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 10:50:57,092] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 10:50:57,092] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]:time (ms) | save-checkpoint: 3141.95 +[default0]: successfully saved checkpoint at iteration 3000 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default5]:[2023-02-16 10:50:57,122] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 10:50:57,122] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3000/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 10:50:57,122] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3000 is ready now! +[default7]: iteration 3001/ 3814 | consumed samples: 1536512 | consumed tokens: 3146776576 | elapsed time per iteration (s): 9.53 | learning rate: 2.175E-04 | global batch size: 512 | lm loss: 3.127077E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 53.707 | TFLOPs: 40.57 | +[default7]: iteration 3002/ 3814 | consumed samples: 1537024 | consumed tokens: 3147825152 | elapsed time per iteration (s): 5.37 | learning rate: 2.172E-04 | global batch size: 512 | lm loss: 3.080599E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.395 | TFLOPs: 72.05 | +[default7]: iteration 3003/ 3814 | consumed samples: 1537536 | consumed tokens: 3148873728 | elapsed time per iteration (s): 5.38 | learning rate: 2.170E-04 | global batch size: 512 | lm loss: 3.108797E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.241 | TFLOPs: 71.94 | +[default7]: iteration 3004/ 3814 | consumed samples: 1538048 | consumed tokens: 3149922304 | elapsed time per iteration (s): 5.37 | learning rate: 2.167E-04 | global batch size: 512 | lm loss: 3.126467E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.322 | TFLOPs: 72.00 | +[default7]: iteration 3005/ 3814 | consumed samples: 1538560 | consumed tokens: 3150970880 | elapsed time per iteration (s): 5.37 | learning rate: 2.164E-04 | global batch size: 512 | lm loss: 3.091886E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.417 | TFLOPs: 72.07 | +[default7]: iteration 3006/ 3814 | consumed samples: 1539072 | consumed tokens: 3152019456 | elapsed time per iteration (s): 5.36 | learning rate: 2.161E-04 | global batch size: 512 | lm loss: 3.110101E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.446 | TFLOPs: 72.09 | +[default7]: iteration 3007/ 3814 | consumed samples: 1539584 | consumed tokens: 3153068032 | elapsed time per iteration (s): 5.36 | learning rate: 2.159E-04 | global batch size: 512 | lm loss: 3.130904E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.559 | TFLOPs: 72.18 | +[default7]: iteration 3008/ 3814 | consumed samples: 1540096 | consumed tokens: 3154116608 | elapsed time per iteration (s): 5.37 | learning rate: 2.156E-04 | global batch size: 512 | lm loss: 3.093498E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.323 | TFLOPs: 72.00 | +[default7]: iteration 3009/ 3814 | consumed samples: 1540608 | consumed tokens: 3155165184 | elapsed time per iteration (s): 5.42 | learning rate: 2.153E-04 | global batch size: 512 | lm loss: 3.120877E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.426 | TFLOPs: 71.32 | +[default7]: iteration 3010/ 3814 | consumed samples: 1541120 | consumed tokens: 3156213760 | elapsed time per iteration (s): 5.42 | learning rate: 2.151E-04 | global batch size: 512 | lm loss: 3.082042E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.538 | TFLOPs: 71.41 | +[default7]: iteration 3011/ 3814 | consumed samples: 1541632 | consumed tokens: 3157262336 | elapsed time per iteration (s): 5.39 | learning rate: 2.148E-04 | global batch size: 512 | lm loss: 3.092675E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.074 | TFLOPs: 71.81 | +[default7]: iteration 3012/ 3814 | consumed samples: 1542144 | consumed tokens: 3158310912 | elapsed time per iteration (s): 5.36 | learning rate: 2.145E-04 | global batch size: 512 | lm loss: 3.123072E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.564 | TFLOPs: 72.18 | +[default7]: iteration 3013/ 3814 | consumed samples: 1542656 | consumed tokens: 3159359488 | elapsed time per iteration (s): 5.40 | learning rate: 2.142E-04 | global batch size: 512 | lm loss: 3.099802E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.832 | TFLOPs: 71.63 | +[default7]: iteration 3014/ 3814 | consumed samples: 1543168 | consumed tokens: 3160408064 | elapsed time per iteration (s): 5.39 | learning rate: 2.140E-04 | global batch size: 512 | lm loss: 3.103224E+00 | loss scale: 262144.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.959 | TFLOPs: 71.72 | +[default7]: iteration 3015/ 3814 | consumed samples: 1543680 | consumed tokens: 3161456640 | elapsed time per iteration (s): 5.40 | learning rate: 2.137E-04 | global batch size: 512 | lm loss: 3.116377E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.763 | TFLOPs: 71.58 | +[default7]: iteration 3016/ 3814 | consumed samples: 1544192 | consumed tokens: 3162505216 | elapsed time per iteration (s): 5.40 | learning rate: 2.134E-04 | global batch size: 512 | lm loss: 3.049976E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.838 | TFLOPs: 71.63 | +[default7]: iteration 3017/ 3814 | consumed samples: 1544704 | consumed tokens: 3163553792 | elapsed time per iteration (s): 5.38 | learning rate: 2.131E-04 | global batch size: 512 | lm loss: 3.066610E+00 | loss scale: 262144.0 | grad norm: 0.105 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.249 | TFLOPs: 71.94 | +[default7]: iteration 3018/ 3814 | consumed samples: 1545216 | consumed tokens: 3164602368 | elapsed time per iteration (s): 5.37 | learning rate: 2.129E-04 | global batch size: 512 | lm loss: 3.060205E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.283 | TFLOPs: 71.97 | +[default7]: iteration 3019/ 3814 | consumed samples: 1545728 | consumed tokens: 3165650944 | elapsed time per iteration (s): 5.35 | learning rate: 2.126E-04 | global batch size: 512 | lm loss: 3.074914E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.685 | TFLOPs: 72.27 | +[default7]: iteration 3020/ 3814 | consumed samples: 1546240 | consumed tokens: 3166699520 | elapsed time per iteration (s): 5.37 | learning rate: 2.123E-04 | global batch size: 512 | lm loss: 3.088223E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.409 | TFLOPs: 72.06 | +[default7]: iteration 3021/ 3814 | consumed samples: 1546752 | consumed tokens: 3167748096 | elapsed time per iteration (s): 5.39 | learning rate: 2.121E-04 | global batch size: 512 | lm loss: 3.101875E+00 | loss scale: 262144.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.078 | TFLOPs: 71.81 | +[default7]: iteration 3022/ 3814 | consumed samples: 1547264 | consumed tokens: 3168796672 | elapsed time per iteration (s): 5.38 | learning rate: 2.118E-04 | global batch size: 512 | lm loss: 3.096799E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.130 | TFLOPs: 71.85 | +[default7]: iteration 3023/ 3814 | consumed samples: 1547776 | consumed tokens: 3169845248 | elapsed time per iteration (s): 5.38 | learning rate: 2.115E-04 | global batch size: 512 | lm loss: 3.089823E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.127 | TFLOPs: 71.85 | +[default7]: iteration 3024/ 3814 | consumed samples: 1548288 | consumed tokens: 3170893824 | elapsed time per iteration (s): 5.35 | learning rate: 2.113E-04 | global batch size: 512 | lm loss: 3.092019E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.615 | TFLOPs: 72.22 | +[default7]: iteration 3025/ 3814 | consumed samples: 1548800 | consumed tokens: 3171942400 | elapsed time per iteration (s): 5.36 | learning rate: 2.110E-04 | global batch size: 512 | lm loss: 3.110866E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.575 | TFLOPs: 72.19 | +[default7]: iteration 3026/ 3814 | consumed samples: 1549312 | consumed tokens: 3172990976 | elapsed time per iteration (s): 5.36 | learning rate: 2.107E-04 | global batch size: 512 | lm loss: 3.112800E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.553 | TFLOPs: 72.17 | +[default7]: iteration 3027/ 3814 | consumed samples: 1549824 | consumed tokens: 3174039552 | elapsed time per iteration (s): 5.36 | learning rate: 2.104E-04 | global batch size: 512 | lm loss: 3.105835E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.607 | TFLOPs: 72.21 | +[default7]: iteration 3028/ 3814 | consumed samples: 1550336 | consumed tokens: 3175088128 | elapsed time per iteration (s): 5.36 | learning rate: 2.102E-04 | global batch size: 512 | lm loss: 3.114540E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.522 | TFLOPs: 72.15 | +[default7]: iteration 3029/ 3814 | consumed samples: 1550848 | consumed tokens: 3176136704 | elapsed time per iteration (s): 5.33 | learning rate: 2.099E-04 | global batch size: 512 | lm loss: 3.085704E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.030 | TFLOPs: 72.53 | +[default7]: iteration 3030/ 3814 | consumed samples: 1551360 | consumed tokens: 3177185280 | elapsed time per iteration (s): 5.35 | learning rate: 2.096E-04 | global batch size: 512 | lm loss: 3.097832E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.633 | TFLOPs: 72.23 | +[default7]: iteration 3031/ 3814 | consumed samples: 1551872 | consumed tokens: 3178233856 | elapsed time per iteration (s): 5.34 | learning rate: 2.094E-04 | global batch size: 512 | lm loss: 3.074950E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.806 | TFLOPs: 72.36 | +[default7]: iteration 3032/ 3814 | consumed samples: 1552384 | consumed tokens: 3179282432 | elapsed time per iteration (s): 5.38 | learning rate: 2.091E-04 | global batch size: 512 | lm loss: 3.070137E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.255 | TFLOPs: 71.95 | +[default7]: iteration 3033/ 3814 | consumed samples: 1552896 | consumed tokens: 3180331008 | elapsed time per iteration (s): 5.37 | learning rate: 2.088E-04 | global batch size: 512 | lm loss: 3.120883E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.348 | TFLOPs: 72.02 | +[default7]: iteration 3034/ 3814 | consumed samples: 1553408 | consumed tokens: 3181379584 | elapsed time per iteration (s): 5.37 | learning rate: 2.086E-04 | global batch size: 512 | lm loss: 3.061470E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.381 | TFLOPs: 72.04 | +[default7]: iteration 3035/ 3814 | consumed samples: 1553920 | consumed tokens: 3182428160 | elapsed time per iteration (s): 5.35 | learning rate: 2.083E-04 | global batch size: 512 | lm loss: 3.075895E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.756 | TFLOPs: 72.33 | +[default7]: iteration 3036/ 3814 | consumed samples: 1554432 | consumed tokens: 3183476736 | elapsed time per iteration (s): 5.38 | learning rate: 2.080E-04 | global batch size: 512 | lm loss: 3.097591E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.094 | TFLOPs: 71.83 | +[default7]: iteration 3037/ 3814 | consumed samples: 1554944 | consumed tokens: 3184525312 | elapsed time per iteration (s): 5.37 | learning rate: 2.078E-04 | global batch size: 512 | lm loss: 3.088645E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.425 | TFLOPs: 72.08 | +[default7]: iteration 3038/ 3814 | consumed samples: 1555456 | consumed tokens: 3185573888 | elapsed time per iteration (s): 5.38 | learning rate: 2.075E-04 | global batch size: 512 | lm loss: 3.104522E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.200 | TFLOPs: 71.91 | +[default7]: iteration 3039/ 3814 | consumed samples: 1555968 | consumed tokens: 3186622464 | elapsed time per iteration (s): 5.36 | learning rate: 2.072E-04 | global batch size: 512 | lm loss: 3.113776E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.555 | TFLOPs: 72.17 | +[default7]: iteration 3040/ 3814 | consumed samples: 1556480 | consumed tokens: 3187671040 | elapsed time per iteration (s): 5.38 | learning rate: 2.070E-04 | global batch size: 512 | lm loss: 3.096279E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.116 | TFLOPs: 71.84 | +[default7]: iteration 3041/ 3814 | consumed samples: 1556992 | consumed tokens: 3188719616 | elapsed time per iteration (s): 5.36 | learning rate: 2.067E-04 | global batch size: 512 | lm loss: 3.121911E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.553 | TFLOPs: 72.17 | +[default7]: iteration 3042/ 3814 | consumed samples: 1557504 | consumed tokens: 3189768192 | elapsed time per iteration (s): 5.35 | learning rate: 2.065E-04 | global batch size: 512 | lm loss: 3.049973E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.728 | TFLOPs: 72.30 | +[default7]: iteration 3043/ 3814 | consumed samples: 1558016 | consumed tokens: 3190816768 | elapsed time per iteration (s): 5.35 | learning rate: 2.062E-04 | global batch size: 512 | lm loss: 3.075810E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.689 | TFLOPs: 72.28 | +[default7]: iteration 3044/ 3814 | consumed samples: 1558528 | consumed tokens: 3191865344 | elapsed time per iteration (s): 5.35 | learning rate: 2.059E-04 | global batch size: 512 | lm loss: 3.060394E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.749 | TFLOPs: 72.32 | +[default7]: iteration 3045/ 3814 | consumed samples: 1559040 | consumed tokens: 3192913920 | elapsed time per iteration (s): 5.38 | learning rate: 2.057E-04 | global batch size: 512 | lm loss: 3.110845E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.146 | TFLOPs: 71.87 | +[default7]: iteration 3046/ 3814 | consumed samples: 1559552 | consumed tokens: 3193962496 | elapsed time per iteration (s): 5.38 | learning rate: 2.054E-04 | global batch size: 512 | lm loss: 3.087710E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.237 | TFLOPs: 71.93 | +[default7]: iteration 3047/ 3814 | consumed samples: 1560064 | consumed tokens: 3195011072 | elapsed time per iteration (s): 5.37 | learning rate: 2.051E-04 | global batch size: 512 | lm loss: 3.093414E+00 | loss scale: 262144.0 | grad norm: 0.107 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.331 | TFLOPs: 72.00 | +[default7]: iteration 3048/ 3814 | consumed samples: 1560576 | consumed tokens: 3196059648 | elapsed time per iteration (s): 5.38 | learning rate: 2.049E-04 | global batch size: 512 | lm loss: 3.109571E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.125 | TFLOPs: 71.85 | +[default7]: iteration 3049/ 3814 | consumed samples: 1561088 | consumed tokens: 3197108224 | elapsed time per iteration (s): 5.38 | learning rate: 2.046E-04 | global batch size: 512 | lm loss: 3.092308E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.232 | TFLOPs: 71.93 | +[default7]: iteration 3050/ 3814 | consumed samples: 1561600 | consumed tokens: 3198156800 | elapsed time per iteration (s): 5.36 | learning rate: 2.044E-04 | global batch size: 512 | lm loss: 3.055957E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.584 | TFLOPs: 72.20 | +[default7]: iteration 3051/ 3814 | consumed samples: 1562112 | consumed tokens: 3199205376 | elapsed time per iteration (s): 5.36 | learning rate: 2.041E-04 | global batch size: 512 | lm loss: 3.082047E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.448 | TFLOPs: 72.09 | +[default7]: iteration 3052/ 3814 | consumed samples: 1562624 | consumed tokens: 3200253952 | elapsed time per iteration (s): 5.36 | learning rate: 2.038E-04 | global batch size: 512 | lm loss: 3.088116E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.511 | TFLOPs: 72.14 | +[default7]: iteration 3053/ 3814 | consumed samples: 1563136 | consumed tokens: 3201302528 | elapsed time per iteration (s): 5.35 | learning rate: 2.036E-04 | global batch size: 512 | lm loss: 3.074938E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.780 | TFLOPs: 72.34 | +[default7]: iteration 3054/ 3814 | consumed samples: 1563648 | consumed tokens: 3202351104 | elapsed time per iteration (s): 5.38 | learning rate: 2.033E-04 | global batch size: 512 | lm loss: 3.086363E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.162 | TFLOPs: 71.88 | +[default7]: iteration 3055/ 3814 | consumed samples: 1564160 | consumed tokens: 3203399680 | elapsed time per iteration (s): 5.38 | learning rate: 2.030E-04 | global batch size: 512 | lm loss: 3.032577E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.091 | TFLOPs: 71.82 | +[default7]: iteration 3056/ 3814 | consumed samples: 1564672 | consumed tokens: 3204448256 | elapsed time per iteration (s): 5.37 | learning rate: 2.028E-04 | global batch size: 512 | lm loss: 3.082378E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.361 | TFLOPs: 72.03 | +[default7]: iteration 3057/ 3814 | consumed samples: 1565184 | consumed tokens: 3205496832 | elapsed time per iteration (s): 5.38 | learning rate: 2.025E-04 | global batch size: 512 | lm loss: 3.085689E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.093 | TFLOPs: 71.83 | +[default7]: iteration 3058/ 3814 | consumed samples: 1565696 | consumed tokens: 3206545408 | elapsed time per iteration (s): 5.36 | learning rate: 2.023E-04 | global batch size: 512 | lm loss: 3.077590E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.539 | TFLOPs: 72.16 | +[default7]: iteration 3059/ 3814 | consumed samples: 1566208 | consumed tokens: 3207593984 | elapsed time per iteration (s): 5.38 | learning rate: 2.020E-04 | global batch size: 512 | lm loss: 3.065764E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.133 | TFLOPs: 71.86 | +[default7]: iteration 3060/ 3814 | consumed samples: 1566720 | consumed tokens: 3208642560 | elapsed time per iteration (s): 5.38 | learning rate: 2.017E-04 | global batch size: 512 | lm loss: 3.111279E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.173 | TFLOPs: 71.89 | +[default7]: iteration 3061/ 3814 | consumed samples: 1567232 | consumed tokens: 3209691136 | elapsed time per iteration (s): 5.37 | learning rate: 2.015E-04 | global batch size: 512 | lm loss: 3.090999E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.419 | TFLOPs: 72.07 | +[default7]: iteration 3062/ 3814 | consumed samples: 1567744 | consumed tokens: 3210739712 | elapsed time per iteration (s): 5.42 | learning rate: 2.012E-04 | global batch size: 512 | lm loss: 3.074444E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.413 | TFLOPs: 71.31 | +[default7]: iteration 3063/ 3814 | consumed samples: 1568256 | consumed tokens: 3211788288 | elapsed time per iteration (s): 5.33 | learning rate: 2.010E-04 | global batch size: 512 | lm loss: 3.057670E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 96.019 | TFLOPs: 72.52 | +[default7]: iteration 3064/ 3814 | consumed samples: 1568768 | consumed tokens: 3212836864 | elapsed time per iteration (s): 5.38 | learning rate: 2.007E-04 | global batch size: 512 | lm loss: 3.105745E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.125 | TFLOPs: 71.85 | +[default7]: iteration 3065/ 3814 | consumed samples: 1569280 | consumed tokens: 3213885440 | elapsed time per iteration (s): 5.39 | learning rate: 2.005E-04 | global batch size: 512 | lm loss: 3.082005E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.007 | TFLOPs: 71.76 | +[default7]: iteration 3066/ 3814 | consumed samples: 1569792 | consumed tokens: 3214934016 | elapsed time per iteration (s): 5.42 | learning rate: 2.002E-04 | global batch size: 512 | lm loss: 3.068962E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.518 | TFLOPs: 71.39 | +[default7]: iteration 3067/ 3814 | consumed samples: 1570304 | consumed tokens: 3215982592 | elapsed time per iteration (s): 5.41 | learning rate: 1.999E-04 | global batch size: 512 | lm loss: 3.084008E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.586 | TFLOPs: 71.44 | +[default7]: iteration 3068/ 3814 | consumed samples: 1570816 | consumed tokens: 3217031168 | elapsed time per iteration (s): 5.40 | learning rate: 1.997E-04 | global batch size: 512 | lm loss: 3.078859E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.885 | TFLOPs: 71.67 | +[default7]: iteration 3069/ 3814 | consumed samples: 1571328 | consumed tokens: 3218079744 | elapsed time per iteration (s): 5.39 | learning rate: 1.994E-04 | global batch size: 512 | lm loss: 3.080268E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.020 | TFLOPs: 71.77 | +[default7]: iteration 3070/ 3814 | consumed samples: 1571840 | consumed tokens: 3219128320 | elapsed time per iteration (s): 5.38 | learning rate: 1.992E-04 | global batch size: 512 | lm loss: 3.087777E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.176 | TFLOPs: 71.89 | +[default7]: iteration 3071/ 3814 | consumed samples: 1572352 | consumed tokens: 3220176896 | elapsed time per iteration (s): 5.38 | learning rate: 1.989E-04 | global batch size: 512 | lm loss: 3.089336E+00 | loss scale: 262144.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.137 | TFLOPs: 71.86 | +[default7]: iteration 3072/ 3814 | consumed samples: 1572864 | consumed tokens: 3221225472 | elapsed time per iteration (s): 5.38 | learning rate: 1.987E-04 | global batch size: 512 | lm loss: 3.026617E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.179 | TFLOPs: 71.89 | +[default7]: iteration 3073/ 3814 | consumed samples: 1573376 | consumed tokens: 3222274048 | elapsed time per iteration (s): 5.36 | learning rate: 1.984E-04 | global batch size: 512 | lm loss: 3.082304E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.448 | TFLOPs: 72.09 | +[default7]: iteration 3074/ 3814 | consumed samples: 1573888 | consumed tokens: 3223322624 | elapsed time per iteration (s): 5.37 | learning rate: 1.981E-04 | global batch size: 512 | lm loss: 3.082640E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.330 | TFLOPs: 72.00 | +[default7]: iteration 3075/ 3814 | consumed samples: 1574400 | consumed tokens: 3224371200 | elapsed time per iteration (s): 5.36 | learning rate: 1.979E-04 | global batch size: 512 | lm loss: 3.059321E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.448 | TFLOPs: 72.09 | +[default7]: iteration 3076/ 3814 | consumed samples: 1574912 | consumed tokens: 3225419776 | elapsed time per iteration (s): 5.38 | learning rate: 1.976E-04 | global batch size: 512 | lm loss: 3.108328E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.196 | TFLOPs: 71.90 | +[default7]: iteration 3077/ 3814 | consumed samples: 1575424 | consumed tokens: 3226468352 | elapsed time per iteration (s): 5.39 | learning rate: 1.974E-04 | global batch size: 512 | lm loss: 3.095368E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.070 | TFLOPs: 71.81 | +[default7]: iteration 3078/ 3814 | consumed samples: 1575936 | consumed tokens: 3227516928 | elapsed time per iteration (s): 5.40 | learning rate: 1.971E-04 | global batch size: 512 | lm loss: 3.073487E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.901 | TFLOPs: 71.68 | +[default7]: iteration 3079/ 3814 | consumed samples: 1576448 | consumed tokens: 3228565504 | elapsed time per iteration (s): 5.35 | learning rate: 1.969E-04 | global batch size: 512 | lm loss: 3.070576E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.620 | TFLOPs: 72.22 | +[default7]: iteration 3080/ 3814 | consumed samples: 1576960 | consumed tokens: 3229614080 | elapsed time per iteration (s): 5.39 | learning rate: 1.966E-04 | global batch size: 512 | lm loss: 3.107008E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.937 | TFLOPs: 71.71 | +[default7]: iteration 3081/ 3814 | consumed samples: 1577472 | consumed tokens: 3230662656 | elapsed time per iteration (s): 5.40 | learning rate: 1.964E-04 | global batch size: 512 | lm loss: 3.065836E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.754 | TFLOPs: 71.57 | +[default7]: iteration 3082/ 3814 | consumed samples: 1577984 | consumed tokens: 3231711232 | elapsed time per iteration (s): 5.38 | learning rate: 1.961E-04 | global batch size: 512 | lm loss: 3.078704E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.218 | TFLOPs: 71.92 | +[default7]: iteration 3083/ 3814 | consumed samples: 1578496 | consumed tokens: 3232759808 | elapsed time per iteration (s): 5.38 | learning rate: 1.959E-04 | global batch size: 512 | lm loss: 3.090041E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.221 | TFLOPs: 71.92 | +[default7]: iteration 3084/ 3814 | consumed samples: 1579008 | consumed tokens: 3233808384 | elapsed time per iteration (s): 5.37 | learning rate: 1.956E-04 | global batch size: 512 | lm loss: 3.081856E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.303 | TFLOPs: 71.98 | +[default7]: iteration 3085/ 3814 | consumed samples: 1579520 | consumed tokens: 3234856960 | elapsed time per iteration (s): 5.37 | learning rate: 1.954E-04 | global batch size: 512 | lm loss: 3.099879E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.426 | TFLOPs: 72.08 | +[default7]: iteration 3086/ 3814 | consumed samples: 1580032 | consumed tokens: 3235905536 | elapsed time per iteration (s): 5.37 | learning rate: 1.951E-04 | global batch size: 512 | lm loss: 3.055310E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.318 | TFLOPs: 71.99 | +[default7]: iteration 3087/ 3814 | consumed samples: 1580544 | consumed tokens: 3236954112 | elapsed time per iteration (s): 5.38 | learning rate: 1.949E-04 | global batch size: 512 | lm loss: 3.072848E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.199 | TFLOPs: 71.91 | +[default7]: iteration 3088/ 3814 | consumed samples: 1581056 | consumed tokens: 3238002688 | elapsed time per iteration (s): 5.38 | learning rate: 1.946E-04 | global batch size: 512 | lm loss: 3.091285E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.094 | TFLOPs: 71.83 | +[default7]: iteration 3089/ 3814 | consumed samples: 1581568 | consumed tokens: 3239051264 | elapsed time per iteration (s): 5.38 | learning rate: 1.944E-04 | global batch size: 512 | lm loss: 3.098967E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.237 | TFLOPs: 71.93 | +[default7]: iteration 3090/ 3814 | consumed samples: 1582080 | consumed tokens: 3240099840 | elapsed time per iteration (s): 5.40 | learning rate: 1.941E-04 | global batch size: 512 | lm loss: 3.070031E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.828 | TFLOPs: 71.62 | +[default7]: iteration 3091/ 3814 | consumed samples: 1582592 | consumed tokens: 3241148416 | elapsed time per iteration (s): 5.37 | learning rate: 1.939E-04 | global batch size: 512 | lm loss: 3.046371E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.301 | TFLOPs: 71.98 | +[default7]: iteration 3092/ 3814 | consumed samples: 1583104 | consumed tokens: 3242196992 | elapsed time per iteration (s): 5.35 | learning rate: 1.936E-04 | global batch size: 512 | lm loss: 3.084754E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.624 | TFLOPs: 72.23 | +[default7]: iteration 3093/ 3814 | consumed samples: 1583616 | consumed tokens: 3243245568 | elapsed time per iteration (s): 5.38 | learning rate: 1.934E-04 | global batch size: 512 | lm loss: 3.072293E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.122 | TFLOPs: 71.85 | +[default7]: iteration 3094/ 3814 | consumed samples: 1584128 | consumed tokens: 3244294144 | elapsed time per iteration (s): 5.39 | learning rate: 1.931E-04 | global batch size: 512 | lm loss: 3.050462E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.062 | TFLOPs: 71.80 | +[default7]: iteration 3095/ 3814 | consumed samples: 1584640 | consumed tokens: 3245342720 | elapsed time per iteration (s): 5.37 | learning rate: 1.929E-04 | global batch size: 512 | lm loss: 3.087982E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.281 | TFLOPs: 71.97 | +[default7]: iteration 3096/ 3814 | consumed samples: 1585152 | consumed tokens: 3246391296 | elapsed time per iteration (s): 5.38 | learning rate: 1.926E-04 | global batch size: 512 | lm loss: 3.074706E+00 | loss scale: 262144.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.107 | TFLOPs: 71.84 | +[default7]: iteration 3097/ 3814 | consumed samples: 1585664 | consumed tokens: 3247439872 | elapsed time per iteration (s): 5.36 | learning rate: 1.924E-04 | global batch size: 512 | lm loss: 3.094036E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.522 | TFLOPs: 72.15 | +[default7]: iteration 3098/ 3814 | consumed samples: 1586176 | consumed tokens: 3248488448 | elapsed time per iteration (s): 5.36 | learning rate: 1.921E-04 | global batch size: 512 | lm loss: 3.082595E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.587 | TFLOPs: 72.20 | +[default7]: iteration 3099/ 3814 | consumed samples: 1586688 | consumed tokens: 3249537024 | elapsed time per iteration (s): 5.36 | learning rate: 1.919E-04 | global batch size: 512 | lm loss: 3.088135E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.492 | TFLOPs: 72.13 | +[default7]: iteration 3100/ 3814 | consumed samples: 1587200 | consumed tokens: 3250585600 | elapsed time per iteration (s): 5.37 | learning rate: 1.916E-04 | global batch size: 512 | lm loss: 3.078251E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.338 | TFLOPs: 72.01 | +[default7]: iteration 3101/ 3814 | consumed samples: 1587712 | consumed tokens: 3251634176 | elapsed time per iteration (s): 5.38 | learning rate: 1.914E-04 | global batch size: 512 | lm loss: 3.113255E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.137 | TFLOPs: 71.86 | +[default7]: iteration 3102/ 3814 | consumed samples: 1588224 | consumed tokens: 3252682752 | elapsed time per iteration (s): 5.38 | learning rate: 1.911E-04 | global batch size: 512 | lm loss: 3.076038E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.252 | TFLOPs: 71.94 | +[default7]: iteration 3103/ 3814 | consumed samples: 1588736 | consumed tokens: 3253731328 | elapsed time per iteration (s): 5.36 | learning rate: 1.909E-04 | global batch size: 512 | lm loss: 3.094991E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.439 | TFLOPs: 72.09 | +[default7]: iteration 3104/ 3814 | consumed samples: 1589248 | consumed tokens: 3254779904 | elapsed time per iteration (s): 5.38 | learning rate: 1.906E-04 | global batch size: 512 | lm loss: 3.065823E+00 | loss scale: 262144.0 | grad norm: 0.108 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.229 | TFLOPs: 71.93 | +[default7]: iteration 3105/ 3814 | consumed samples: 1589760 | consumed tokens: 3255828480 | elapsed time per iteration (s): 5.37 | learning rate: 1.904E-04 | global batch size: 512 | lm loss: 3.077650E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.350 | TFLOPs: 72.02 | +[default7]: iteration 3106/ 3814 | consumed samples: 1590272 | consumed tokens: 3256877056 | elapsed time per iteration (s): 5.36 | learning rate: 1.901E-04 | global batch size: 512 | lm loss: 3.088267E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.597 | TFLOPs: 72.21 | +[default7]: iteration 3107/ 3814 | consumed samples: 1590784 | consumed tokens: 3257925632 | elapsed time per iteration (s): 5.37 | learning rate: 1.899E-04 | global batch size: 512 | lm loss: 3.080904E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.266 | TFLOPs: 71.96 | +[default7]: iteration 3108/ 3814 | consumed samples: 1591296 | consumed tokens: 3258974208 | elapsed time per iteration (s): 5.37 | learning rate: 1.897E-04 | global batch size: 512 | lm loss: 3.082758E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.316 | TFLOPs: 71.99 | +[default7]: iteration 3109/ 3814 | consumed samples: 1591808 | consumed tokens: 3260022784 | elapsed time per iteration (s): 5.38 | learning rate: 1.894E-04 | global batch size: 512 | lm loss: 3.076950E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.197 | TFLOPs: 71.90 | +[default7]: iteration 3110/ 3814 | consumed samples: 1592320 | consumed tokens: 3261071360 | elapsed time per iteration (s): 5.37 | learning rate: 1.892E-04 | global batch size: 512 | lm loss: 3.026777E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.291 | TFLOPs: 71.97 | +[default7]: iteration 3111/ 3814 | consumed samples: 1592832 | consumed tokens: 3262119936 | elapsed time per iteration (s): 5.38 | learning rate: 1.889E-04 | global batch size: 512 | lm loss: 3.082577E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.129 | TFLOPs: 71.85 | +[default7]: iteration 3112/ 3814 | consumed samples: 1593344 | consumed tokens: 3263168512 | elapsed time per iteration (s): 5.40 | learning rate: 1.887E-04 | global batch size: 512 | lm loss: 3.063827E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.879 | TFLOPs: 71.66 | +[default7]: iteration 3113/ 3814 | consumed samples: 1593856 | consumed tokens: 3264217088 | elapsed time per iteration (s): 5.35 | learning rate: 1.884E-04 | global batch size: 512 | lm loss: 3.058010E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.776 | TFLOPs: 72.34 | +[default7]: iteration 3114/ 3814 | consumed samples: 1594368 | consumed tokens: 3265265664 | elapsed time per iteration (s): 5.41 | learning rate: 1.882E-04 | global batch size: 512 | lm loss: 3.081609E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.632 | TFLOPs: 71.48 | +[default7]: iteration 3115/ 3814 | consumed samples: 1594880 | consumed tokens: 3266314240 | elapsed time per iteration (s): 5.39 | learning rate: 1.879E-04 | global batch size: 512 | lm loss: 3.073005E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.922 | TFLOPs: 71.70 | +[default7]: iteration 3116/ 3814 | consumed samples: 1595392 | consumed tokens: 3267362816 | elapsed time per iteration (s): 5.38 | learning rate: 1.877E-04 | global batch size: 512 | lm loss: 3.064907E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.226 | TFLOPs: 71.93 | +[default7]: iteration 3117/ 3814 | consumed samples: 1595904 | consumed tokens: 3268411392 | elapsed time per iteration (s): 5.40 | learning rate: 1.875E-04 | global batch size: 512 | lm loss: 3.040088E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.738 | TFLOPs: 71.56 | +[default7]: iteration 3118/ 3814 | consumed samples: 1596416 | consumed tokens: 3269459968 | elapsed time per iteration (s): 5.38 | learning rate: 1.872E-04 | global batch size: 512 | lm loss: 3.105173E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.200 | TFLOPs: 71.91 | +[default7]: iteration 3119/ 3814 | consumed samples: 1596928 | consumed tokens: 3270508544 | elapsed time per iteration (s): 5.37 | learning rate: 1.870E-04 | global batch size: 512 | lm loss: 3.098557E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.372 | TFLOPs: 72.04 | +[default7]: iteration 3120/ 3814 | consumed samples: 1597440 | consumed tokens: 3271557120 | elapsed time per iteration (s): 5.37 | learning rate: 1.867E-04 | global batch size: 512 | lm loss: 3.079145E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.301 | TFLOPs: 71.98 | +[default7]: iteration 3121/ 3814 | consumed samples: 1597952 | consumed tokens: 3272605696 | elapsed time per iteration (s): 5.35 | learning rate: 1.865E-04 | global batch size: 512 | lm loss: 3.065713E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.740 | TFLOPs: 72.31 | +[default7]: iteration 3122/ 3814 | consumed samples: 1598464 | consumed tokens: 3273654272 | elapsed time per iteration (s): 5.36 | learning rate: 1.863E-04 | global batch size: 512 | lm loss: 3.069606E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.533 | TFLOPs: 72.16 | +[default7]: iteration 3123/ 3814 | consumed samples: 1598976 | consumed tokens: 3274702848 | elapsed time per iteration (s): 5.35 | learning rate: 1.860E-04 | global batch size: 512 | lm loss: 3.111257E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.737 | TFLOPs: 72.31 | +[default7]: iteration 3124/ 3814 | consumed samples: 1599488 | consumed tokens: 3275751424 | elapsed time per iteration (s): 5.38 | learning rate: 1.858E-04 | global batch size: 512 | lm loss: 3.059157E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.148 | TFLOPs: 71.87 | +[default7]: iteration 3125/ 3814 | consumed samples: 1600000 | consumed tokens: 3276800000 | elapsed time per iteration (s): 5.37 | learning rate: 1.855E-04 | global batch size: 512 | lm loss: 3.077332E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.368 | TFLOPs: 72.03 | +[default7]: iteration 3126/ 3814 | consumed samples: 1600512 | consumed tokens: 3277848576 | elapsed time per iteration (s): 5.37 | learning rate: 1.853E-04 | global batch size: 512 | lm loss: 3.058561E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.266 | TFLOPs: 71.96 | +[default7]: iteration 3127/ 3814 | consumed samples: 1601024 | consumed tokens: 3278897152 | elapsed time per iteration (s): 5.38 | learning rate: 1.851E-04 | global batch size: 512 | lm loss: 3.082854E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.201 | TFLOPs: 71.91 | +[default7]: iteration 3128/ 3814 | consumed samples: 1601536 | consumed tokens: 3279945728 | elapsed time per iteration (s): 5.37 | learning rate: 1.848E-04 | global batch size: 512 | lm loss: 3.065570E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.315 | TFLOPs: 71.99 | +[default7]: iteration 3129/ 3814 | consumed samples: 1602048 | consumed tokens: 3280994304 | elapsed time per iteration (s): 5.38 | learning rate: 1.846E-04 | global batch size: 512 | lm loss: 3.077688E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.223 | TFLOPs: 71.92 | +[default7]: iteration 3130/ 3814 | consumed samples: 1602560 | consumed tokens: 3282042880 | elapsed time per iteration (s): 5.36 | learning rate: 1.843E-04 | global batch size: 512 | lm loss: 3.038421E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.515 | TFLOPs: 72.14 | +[default7]: iteration 3131/ 3814 | consumed samples: 1603072 | consumed tokens: 3283091456 | elapsed time per iteration (s): 5.39 | learning rate: 1.841E-04 | global batch size: 512 | lm loss: 3.099950E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.999 | TFLOPs: 71.75 | +[default7]: iteration 3132/ 3814 | consumed samples: 1603584 | consumed tokens: 3284140032 | elapsed time per iteration (s): 5.35 | learning rate: 1.839E-04 | global batch size: 512 | lm loss: 3.075674E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.687 | TFLOPs: 72.27 | +[default7]: iteration 3133/ 3814 | consumed samples: 1604096 | consumed tokens: 3285188608 | elapsed time per iteration (s): 5.39 | learning rate: 1.836E-04 | global batch size: 512 | lm loss: 3.110675E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.944 | TFLOPs: 71.71 | +[default7]: iteration 3134/ 3814 | consumed samples: 1604608 | consumed tokens: 3286237184 | elapsed time per iteration (s): 5.41 | learning rate: 1.834E-04 | global batch size: 512 | lm loss: 3.089610E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.694 | TFLOPs: 71.52 | +[default7]: iteration 3135/ 3814 | consumed samples: 1605120 | consumed tokens: 3287285760 | elapsed time per iteration (s): 5.39 | learning rate: 1.831E-04 | global batch size: 512 | lm loss: 3.101942E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.914 | TFLOPs: 71.69 | +[default7]: iteration 3136/ 3814 | consumed samples: 1605632 | consumed tokens: 3288334336 | elapsed time per iteration (s): 5.39 | learning rate: 1.829E-04 | global batch size: 512 | lm loss: 3.046597E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.998 | TFLOPs: 71.75 | +[default7]: iteration 3137/ 3814 | consumed samples: 1606144 | consumed tokens: 3289382912 | elapsed time per iteration (s): 5.38 | learning rate: 1.827E-04 | global batch size: 512 | lm loss: 3.045756E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.084 | TFLOPs: 71.82 | +[default7]: iteration 3138/ 3814 | consumed samples: 1606656 | consumed tokens: 3290431488 | elapsed time per iteration (s): 5.38 | learning rate: 1.824E-04 | global batch size: 512 | lm loss: 3.063776E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.240 | TFLOPs: 71.94 | +[default7]: iteration 3139/ 3814 | consumed samples: 1607168 | consumed tokens: 3291480064 | elapsed time per iteration (s): 5.40 | learning rate: 1.822E-04 | global batch size: 512 | lm loss: 3.083973E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.902 | TFLOPs: 71.68 | +[default7]: iteration 3140/ 3814 | consumed samples: 1607680 | consumed tokens: 3292528640 | elapsed time per iteration (s): 5.38 | learning rate: 1.820E-04 | global batch size: 512 | lm loss: 3.086459E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.160 | TFLOPs: 71.87 | +[default7]: iteration 3141/ 3814 | consumed samples: 1608192 | consumed tokens: 3293577216 | elapsed time per iteration (s): 5.38 | learning rate: 1.817E-04 | global batch size: 512 | lm loss: 3.031136E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.136 | TFLOPs: 71.86 | +[default7]: iteration 3142/ 3814 | consumed samples: 1608704 | consumed tokens: 3294625792 | elapsed time per iteration (s): 5.38 | learning rate: 1.815E-04 | global batch size: 512 | lm loss: 3.063560E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.124 | TFLOPs: 71.85 | +[default7]: iteration 3143/ 3814 | consumed samples: 1609216 | consumed tokens: 3295674368 | elapsed time per iteration (s): 5.41 | learning rate: 1.813E-04 | global batch size: 512 | lm loss: 3.113611E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.607 | TFLOPs: 71.46 | +[default7]: iteration 3144/ 3814 | consumed samples: 1609728 | consumed tokens: 3296722944 | elapsed time per iteration (s): 5.39 | learning rate: 1.810E-04 | global batch size: 512 | lm loss: 3.057863E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.986 | TFLOPs: 71.74 | +[default7]: iteration 3145/ 3814 | consumed samples: 1610240 | consumed tokens: 3297771520 | elapsed time per iteration (s): 5.40 | learning rate: 1.808E-04 | global batch size: 512 | lm loss: 3.080842E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.874 | TFLOPs: 71.66 | +[default7]: iteration 3146/ 3814 | consumed samples: 1610752 | consumed tokens: 3298820096 | elapsed time per iteration (s): 5.40 | learning rate: 1.806E-04 | global batch size: 512 | lm loss: 3.106575E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.895 | TFLOPs: 71.68 | +[default7]: iteration 3147/ 3814 | consumed samples: 1611264 | consumed tokens: 3299868672 | elapsed time per iteration (s): 5.38 | learning rate: 1.803E-04 | global batch size: 512 | lm loss: 3.071527E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.113 | TFLOPs: 71.84 | +[default7]: iteration 3148/ 3814 | consumed samples: 1611776 | consumed tokens: 3300917248 | elapsed time per iteration (s): 5.41 | learning rate: 1.801E-04 | global batch size: 512 | lm loss: 3.078636E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 3149/ 3814 | consumed samples: 1612288 | consumed tokens: 3301965824 | elapsed time per iteration (s): 5.40 | learning rate: 1.799E-04 | global batch size: 512 | lm loss: 3.044755E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.791 | TFLOPs: 71.60 | +[default7]: iteration 3150/ 3814 | consumed samples: 1612800 | consumed tokens: 3303014400 | elapsed time per iteration (s): 5.38 | learning rate: 1.796E-04 | global batch size: 512 | lm loss: 3.047281E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.097 | TFLOPs: 71.83 | +[default7]: iteration 3151/ 3814 | consumed samples: 1613312 | consumed tokens: 3304062976 | elapsed time per iteration (s): 5.39 | learning rate: 1.794E-04 | global batch size: 512 | lm loss: 3.067882E+00 | loss scale: 262144.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.013 | TFLOPs: 71.76 | +[default7]: iteration 3152/ 3814 | consumed samples: 1613824 | consumed tokens: 3305111552 | elapsed time per iteration (s): 5.39 | learning rate: 1.792E-04 | global batch size: 512 | lm loss: 3.072110E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.961 | TFLOPs: 71.73 | +[default7]: iteration 3153/ 3814 | consumed samples: 1614336 | consumed tokens: 3306160128 | elapsed time per iteration (s): 5.39 | learning rate: 1.789E-04 | global batch size: 512 | lm loss: 3.129831E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.948 | TFLOPs: 71.71 | +[default7]: iteration 3154/ 3814 | consumed samples: 1614848 | consumed tokens: 3307208704 | elapsed time per iteration (s): 5.39 | learning rate: 1.787E-04 | global batch size: 512 | lm loss: 3.030310E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.962 | TFLOPs: 71.73 | +[default7]: iteration 3155/ 3814 | consumed samples: 1615360 | consumed tokens: 3308257280 | elapsed time per iteration (s): 5.40 | learning rate: 1.785E-04 | global batch size: 512 | lm loss: 3.066009E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.871 | TFLOPs: 71.66 | +[default7]: iteration 3156/ 3814 | consumed samples: 1615872 | consumed tokens: 3309305856 | elapsed time per iteration (s): 5.40 | learning rate: 1.782E-04 | global batch size: 512 | lm loss: 3.035276E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.745 | TFLOPs: 71.56 | +[default7]: iteration 3157/ 3814 | consumed samples: 1616384 | consumed tokens: 3310354432 | elapsed time per iteration (s): 5.39 | learning rate: 1.780E-04 | global batch size: 512 | lm loss: 3.041964E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.037 | TFLOPs: 71.78 | +[default7]: iteration 3158/ 3814 | consumed samples: 1616896 | consumed tokens: 3311403008 | elapsed time per iteration (s): 5.39 | learning rate: 1.778E-04 | global batch size: 512 | lm loss: 3.110928E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.987 | TFLOPs: 71.74 | +[default7]: iteration 3159/ 3814 | consumed samples: 1617408 | consumed tokens: 3312451584 | elapsed time per iteration (s): 5.40 | learning rate: 1.776E-04 | global batch size: 512 | lm loss: 3.072273E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.842 | TFLOPs: 71.64 | +[default7]: iteration 3160/ 3814 | consumed samples: 1617920 | consumed tokens: 3313500160 | elapsed time per iteration (s): 5.38 | learning rate: 1.773E-04 | global batch size: 512 | lm loss: 3.049694E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.104 | TFLOPs: 71.83 | +[default7]: iteration 3161/ 3814 | consumed samples: 1618432 | consumed tokens: 3314548736 | elapsed time per iteration (s): 5.41 | learning rate: 1.771E-04 | global batch size: 512 | lm loss: 3.081012E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.612 | TFLOPs: 71.46 | +[default7]: iteration 3162/ 3814 | consumed samples: 1618944 | consumed tokens: 3315597312 | elapsed time per iteration (s): 5.39 | learning rate: 1.769E-04 | global batch size: 512 | lm loss: 3.099621E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.925 | TFLOPs: 71.70 | +[default7]: iteration 3163/ 3814 | consumed samples: 1619456 | consumed tokens: 3316645888 | elapsed time per iteration (s): 5.38 | learning rate: 1.766E-04 | global batch size: 512 | lm loss: 3.059997E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.102 | TFLOPs: 71.83 | +[default7]: iteration 3164/ 3814 | consumed samples: 1619968 | consumed tokens: 3317694464 | elapsed time per iteration (s): 5.38 | learning rate: 1.764E-04 | global batch size: 512 | lm loss: 3.070084E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.204 | TFLOPs: 71.91 | +[default7]: iteration 3165/ 3814 | consumed samples: 1620480 | consumed tokens: 3318743040 | elapsed time per iteration (s): 5.38 | learning rate: 1.762E-04 | global batch size: 512 | lm loss: 3.071957E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.113 | TFLOPs: 71.84 | +[default7]: iteration 3166/ 3814 | consumed samples: 1620992 | consumed tokens: 3319791616 | elapsed time per iteration (s): 5.39 | learning rate: 1.760E-04 | global batch size: 512 | lm loss: 3.063127E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.944 | TFLOPs: 71.71 | +[default7]: iteration 3167/ 3814 | consumed samples: 1621504 | consumed tokens: 3320840192 | elapsed time per iteration (s): 5.38 | learning rate: 1.757E-04 | global batch size: 512 | lm loss: 3.066170E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.241 | TFLOPs: 71.94 | +[default7]: iteration 3168/ 3814 | consumed samples: 1622016 | consumed tokens: 3321888768 | elapsed time per iteration (s): 5.37 | learning rate: 1.755E-04 | global batch size: 512 | lm loss: 3.064512E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.382 | TFLOPs: 72.04 | +[default7]: iteration 3169/ 3814 | consumed samples: 1622528 | consumed tokens: 3322937344 | elapsed time per iteration (s): 5.38 | learning rate: 1.753E-04 | global batch size: 512 | lm loss: 3.086855E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.245 | TFLOPs: 71.94 | +[default7]: iteration 3170/ 3814 | consumed samples: 1623040 | consumed tokens: 3323985920 | elapsed time per iteration (s): 5.38 | learning rate: 1.750E-04 | global batch size: 512 | lm loss: 3.077011E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.247 | TFLOPs: 71.94 | +[default7]: iteration 3171/ 3814 | consumed samples: 1623552 | consumed tokens: 3325034496 | elapsed time per iteration (s): 5.38 | learning rate: 1.748E-04 | global batch size: 512 | lm loss: 3.066883E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.225 | TFLOPs: 71.92 | +[default7]: iteration 3172/ 3814 | consumed samples: 1624064 | consumed tokens: 3326083072 | elapsed time per iteration (s): 5.39 | learning rate: 1.746E-04 | global batch size: 512 | lm loss: 3.081721E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.952 | TFLOPs: 71.72 | +[default7]: iteration 3173/ 3814 | consumed samples: 1624576 | consumed tokens: 3327131648 | elapsed time per iteration (s): 5.38 | learning rate: 1.744E-04 | global batch size: 512 | lm loss: 3.051471E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.226 | TFLOPs: 71.93 | +[default7]: iteration 3174/ 3814 | consumed samples: 1625088 | consumed tokens: 3328180224 | elapsed time per iteration (s): 5.40 | learning rate: 1.741E-04 | global batch size: 512 | lm loss: 3.084188E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.778 | TFLOPs: 71.59 | +[default7]: iteration 3175/ 3814 | consumed samples: 1625600 | consumed tokens: 3329228800 | elapsed time per iteration (s): 5.39 | learning rate: 1.739E-04 | global batch size: 512 | lm loss: 3.054967E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.947 | TFLOPs: 71.71 | +[default7]: iteration 3176/ 3814 | consumed samples: 1626112 | consumed tokens: 3330277376 | elapsed time per iteration (s): 5.38 | learning rate: 1.737E-04 | global batch size: 512 | lm loss: 3.044844E+00 | loss scale: 262144.0 | grad norm: 0.126 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.128 | TFLOPs: 71.85 | +[default7]: iteration 3177/ 3814 | consumed samples: 1626624 | consumed tokens: 3331325952 | elapsed time per iteration (s): 5.40 | learning rate: 1.735E-04 | global batch size: 512 | lm loss: 3.051356E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.770 | TFLOPs: 71.58 | +[default7]: iteration 3178/ 3814 | consumed samples: 1627136 | consumed tokens: 3332374528 | elapsed time per iteration (s): 5.40 | learning rate: 1.732E-04 | global batch size: 512 | lm loss: 3.068130E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.878 | TFLOPs: 71.66 | +[default7]: iteration 3179/ 3814 | consumed samples: 1627648 | consumed tokens: 3333423104 | elapsed time per iteration (s): 5.39 | learning rate: 1.730E-04 | global batch size: 512 | lm loss: 3.066049E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.942 | TFLOPs: 71.71 | +[default7]: iteration 3180/ 3814 | consumed samples: 1628160 | consumed tokens: 3334471680 | elapsed time per iteration (s): 5.40 | learning rate: 1.728E-04 | global batch size: 512 | lm loss: 3.054650E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.813 | TFLOPs: 71.61 | +[default7]: iteration 3181/ 3814 | consumed samples: 1628672 | consumed tokens: 3335520256 | elapsed time per iteration (s): 5.40 | learning rate: 1.726E-04 | global batch size: 512 | lm loss: 3.086224E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.735 | TFLOPs: 71.55 | +[default7]: iteration 3182/ 3814 | consumed samples: 1629184 | consumed tokens: 3336568832 | elapsed time per iteration (s): 5.38 | learning rate: 1.724E-04 | global batch size: 512 | lm loss: 3.078992E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.192 | TFLOPs: 71.90 | +[default7]: iteration 3183/ 3814 | consumed samples: 1629696 | consumed tokens: 3337617408 | elapsed time per iteration (s): 5.37 | learning rate: 1.721E-04 | global batch size: 512 | lm loss: 3.073938E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.340 | TFLOPs: 72.01 | +[default7]: iteration 3184/ 3814 | consumed samples: 1630208 | consumed tokens: 3338665984 | elapsed time per iteration (s): 5.39 | learning rate: 1.719E-04 | global batch size: 512 | lm loss: 3.054662E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.074 | TFLOPs: 71.81 | +[default7]: iteration 3185/ 3814 | consumed samples: 1630720 | consumed tokens: 3339714560 | elapsed time per iteration (s): 5.39 | learning rate: 1.717E-04 | global batch size: 512 | lm loss: 3.090271E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.987 | TFLOPs: 71.74 | +[default7]: iteration 3186/ 3814 | consumed samples: 1631232 | consumed tokens: 3340763136 | elapsed time per iteration (s): 5.38 | learning rate: 1.715E-04 | global batch size: 512 | lm loss: 3.093076E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.157 | TFLOPs: 71.87 | +[default7]: iteration 3187/ 3814 | consumed samples: 1631744 | consumed tokens: 3341811712 | elapsed time per iteration (s): 5.38 | learning rate: 1.712E-04 | global batch size: 512 | lm loss: 3.047122E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.134 | TFLOPs: 71.86 | +[default7]: iteration 3188/ 3814 | consumed samples: 1632256 | consumed tokens: 3342860288 | elapsed time per iteration (s): 5.38 | learning rate: 1.710E-04 | global batch size: 512 | lm loss: 3.024630E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.100 | TFLOPs: 71.83 | +[default7]: iteration 3189/ 3814 | consumed samples: 1632768 | consumed tokens: 3343908864 | elapsed time per iteration (s): 5.36 | learning rate: 1.708E-04 | global batch size: 512 | lm loss: 3.053432E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.438 | TFLOPs: 72.09 | +[default7]: iteration 3190/ 3814 | consumed samples: 1633280 | consumed tokens: 3344957440 | elapsed time per iteration (s): 5.38 | learning rate: 1.706E-04 | global batch size: 512 | lm loss: 3.089203E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.135 | TFLOPs: 71.86 | +[default7]: iteration 3191/ 3814 | consumed samples: 1633792 | consumed tokens: 3346006016 | elapsed time per iteration (s): 5.36 | learning rate: 1.704E-04 | global batch size: 512 | lm loss: 3.053472E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.512 | TFLOPs: 72.14 | +[default7]: iteration 3192/ 3814 | consumed samples: 1634304 | consumed tokens: 3347054592 | elapsed time per iteration (s): 5.37 | learning rate: 1.701E-04 | global batch size: 512 | lm loss: 3.084276E+00 | loss scale: 262144.0 | grad norm: 0.130 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.403 | TFLOPs: 72.06 | +[default7]: iteration 3193/ 3814 | consumed samples: 1634816 | consumed tokens: 3348103168 | elapsed time per iteration (s): 5.38 | learning rate: 1.699E-04 | global batch size: 512 | lm loss: 3.073125E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.189 | TFLOPs: 71.90 | +[default7]: iteration 3194/ 3814 | consumed samples: 1635328 | consumed tokens: 3349151744 | elapsed time per iteration (s): 5.38 | learning rate: 1.697E-04 | global batch size: 512 | lm loss: 3.065445E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.131 | TFLOPs: 71.85 | +[default7]: iteration 3195/ 3814 | consumed samples: 1635840 | consumed tokens: 3350200320 | elapsed time per iteration (s): 5.39 | learning rate: 1.695E-04 | global batch size: 512 | lm loss: 3.082270E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.055 | TFLOPs: 71.80 | +[default7]: iteration 3196/ 3814 | consumed samples: 1636352 | consumed tokens: 3351248896 | elapsed time per iteration (s): 5.39 | learning rate: 1.693E-04 | global batch size: 512 | lm loss: 3.055686E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.928 | TFLOPs: 71.70 | +[default7]: iteration 3197/ 3814 | consumed samples: 1636864 | consumed tokens: 3352297472 | elapsed time per iteration (s): 5.40 | learning rate: 1.691E-04 | global batch size: 512 | lm loss: 3.080547E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.866 | TFLOPs: 71.65 | +[default7]: iteration 3198/ 3814 | consumed samples: 1637376 | consumed tokens: 3353346048 | elapsed time per iteration (s): 5.41 | learning rate: 1.688E-04 | global batch size: 512 | lm loss: 3.030853E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.711 | TFLOPs: 71.54 | +[default7]: iteration 3199/ 3814 | consumed samples: 1637888 | consumed tokens: 3354394624 | elapsed time per iteration (s): 5.38 | learning rate: 1.686E-04 | global batch size: 512 | lm loss: 3.087323E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.113 | TFLOPs: 71.84 | +[default7]: iteration 3200/ 3814 | consumed samples: 1638400 | consumed tokens: 3355443200 | elapsed time per iteration (s): 5.38 | learning rate: 1.684E-04 | global batch size: 512 | lm loss: 3.059324E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.224 | TFLOPs: 71.92 | +[default7]: iteration 3201/ 3814 | consumed samples: 1638912 | consumed tokens: 3356491776 | elapsed time per iteration (s): 5.39 | learning rate: 1.682E-04 | global batch size: 512 | lm loss: 3.077872E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.959 | TFLOPs: 71.72 | +[default7]: iteration 3202/ 3814 | consumed samples: 1639424 | consumed tokens: 3357540352 | elapsed time per iteration (s): 5.38 | learning rate: 1.680E-04 | global batch size: 512 | lm loss: 3.083153E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.255 | TFLOPs: 71.95 | +[default7]: iteration 3203/ 3814 | consumed samples: 1639936 | consumed tokens: 3358588928 | elapsed time per iteration (s): 5.40 | learning rate: 1.678E-04 | global batch size: 512 | lm loss: 3.046932E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.793 | TFLOPs: 71.60 | +[default7]: iteration 3204/ 3814 | consumed samples: 1640448 | consumed tokens: 3359637504 | elapsed time per iteration (s): 5.41 | learning rate: 1.675E-04 | global batch size: 512 | lm loss: 3.053340E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.568 | TFLOPs: 71.43 | +[default7]: iteration 3205/ 3814 | consumed samples: 1640960 | consumed tokens: 3360686080 | elapsed time per iteration (s): 5.39 | learning rate: 1.673E-04 | global batch size: 512 | lm loss: 3.095978E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.972 | TFLOPs: 71.73 | +[default7]: iteration 3206/ 3814 | consumed samples: 1641472 | consumed tokens: 3361734656 | elapsed time per iteration (s): 5.41 | learning rate: 1.671E-04 | global batch size: 512 | lm loss: 3.057784E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.650 | TFLOPs: 71.49 | +[default7]: iteration 3207/ 3814 | consumed samples: 1641984 | consumed tokens: 3362783232 | elapsed time per iteration (s): 5.40 | learning rate: 1.669E-04 | global batch size: 512 | lm loss: 3.030892E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.881 | TFLOPs: 71.66 | +[default7]: iteration 3208/ 3814 | consumed samples: 1642496 | consumed tokens: 3363831808 | elapsed time per iteration (s): 5.43 | learning rate: 1.667E-04 | global batch size: 512 | lm loss: 3.067023E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.279 | TFLOPs: 71.21 | +[default7]: iteration 3209/ 3814 | consumed samples: 1643008 | consumed tokens: 3364880384 | elapsed time per iteration (s): 5.42 | learning rate: 1.665E-04 | global batch size: 512 | lm loss: 3.078065E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.481 | TFLOPs: 71.36 | +[default7]: iteration 3210/ 3814 | consumed samples: 1643520 | consumed tokens: 3365928960 | elapsed time per iteration (s): 5.44 | learning rate: 1.662E-04 | global batch size: 512 | lm loss: 3.065407E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.204 | TFLOPs: 71.15 | +[default7]: iteration 3211/ 3814 | consumed samples: 1644032 | consumed tokens: 3366977536 | elapsed time per iteration (s): 5.42 | learning rate: 1.660E-04 | global batch size: 512 | lm loss: 3.046875E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.524 | TFLOPs: 71.39 | +[default7]: iteration 3212/ 3814 | consumed samples: 1644544 | consumed tokens: 3368026112 | elapsed time per iteration (s): 5.43 | learning rate: 1.658E-04 | global batch size: 512 | lm loss: 3.074739E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.330 | TFLOPs: 71.25 | +[default7]: iteration 3213/ 3814 | consumed samples: 1645056 | consumed tokens: 3369074688 | elapsed time per iteration (s): 5.37 | learning rate: 1.656E-04 | global batch size: 512 | lm loss: 3.080736E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.324 | TFLOPs: 72.00 | +[default7]: iteration 3214/ 3814 | consumed samples: 1645568 | consumed tokens: 3370123264 | elapsed time per iteration (s): 5.40 | learning rate: 1.654E-04 | global batch size: 512 | lm loss: 3.059855E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.782 | TFLOPs: 71.59 | +[default7]: iteration 3215/ 3814 | consumed samples: 1646080 | consumed tokens: 3371171840 | elapsed time per iteration (s): 5.39 | learning rate: 1.652E-04 | global batch size: 512 | lm loss: 3.048213E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.906 | TFLOPs: 71.68 | +[default7]: iteration 3216/ 3814 | consumed samples: 1646592 | consumed tokens: 3372220416 | elapsed time per iteration (s): 5.39 | learning rate: 1.650E-04 | global batch size: 512 | lm loss: 3.038041E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.060 | TFLOPs: 71.80 | +[default7]: iteration 3217/ 3814 | consumed samples: 1647104 | consumed tokens: 3373268992 | elapsed time per iteration (s): 5.38 | learning rate: 1.648E-04 | global batch size: 512 | lm loss: 3.088600E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.164 | TFLOPs: 71.88 | +[default7]: iteration 3218/ 3814 | consumed samples: 1647616 | consumed tokens: 3374317568 | elapsed time per iteration (s): 5.38 | learning rate: 1.646E-04 | global batch size: 512 | lm loss: 3.055095E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.190 | TFLOPs: 71.90 | +[default7]: iteration 3219/ 3814 | consumed samples: 1648128 | consumed tokens: 3375366144 | elapsed time per iteration (s): 5.39 | learning rate: 1.643E-04 | global batch size: 512 | lm loss: 3.081161E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.913 | TFLOPs: 71.69 | +[default7]: iteration 3220/ 3814 | consumed samples: 1648640 | consumed tokens: 3376414720 | elapsed time per iteration (s): 5.37 | learning rate: 1.641E-04 | global batch size: 512 | lm loss: 3.043944E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.257 | TFLOPs: 71.95 | +[default7]: iteration 3221/ 3814 | consumed samples: 1649152 | consumed tokens: 3377463296 | elapsed time per iteration (s): 5.37 | learning rate: 1.639E-04 | global batch size: 512 | lm loss: 3.059099E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.332 | TFLOPs: 72.00 | +[default7]: iteration 3222/ 3814 | consumed samples: 1649664 | consumed tokens: 3378511872 | elapsed time per iteration (s): 5.40 | learning rate: 1.637E-04 | global batch size: 512 | lm loss: 3.048468E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.887 | TFLOPs: 71.67 | +[default7]: iteration 3223/ 3814 | consumed samples: 1650176 | consumed tokens: 3379560448 | elapsed time per iteration (s): 5.40 | learning rate: 1.635E-04 | global batch size: 512 | lm loss: 3.079402E+00 | loss scale: 262144.0 | grad norm: 0.124 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.855 | TFLOPs: 71.65 | +[default7]: iteration 3224/ 3814 | consumed samples: 1650688 | consumed tokens: 3380609024 | elapsed time per iteration (s): 5.38 | learning rate: 1.633E-04 | global batch size: 512 | lm loss: 3.058895E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.101 | TFLOPs: 71.83 | +[default7]: iteration 3225/ 3814 | consumed samples: 1651200 | consumed tokens: 3381657600 | elapsed time per iteration (s): 5.40 | learning rate: 1.631E-04 | global batch size: 512 | lm loss: 3.081516E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.844 | TFLOPs: 71.64 | +[default7]: iteration 3226/ 3814 | consumed samples: 1651712 | consumed tokens: 3382706176 | elapsed time per iteration (s): 5.39 | learning rate: 1.629E-04 | global batch size: 512 | lm loss: 3.058286E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.968 | TFLOPs: 71.73 | +[default7]: iteration 3227/ 3814 | consumed samples: 1652224 | consumed tokens: 3383754752 | elapsed time per iteration (s): 5.38 | learning rate: 1.627E-04 | global batch size: 512 | lm loss: 3.063231E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.208 | TFLOPs: 71.91 | +[default7]: iteration 3228/ 3814 | consumed samples: 1652736 | consumed tokens: 3384803328 | elapsed time per iteration (s): 5.40 | learning rate: 1.625E-04 | global batch size: 512 | lm loss: 3.032269E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.866 | TFLOPs: 71.65 | +[default7]: iteration 3229/ 3814 | consumed samples: 1653248 | consumed tokens: 3385851904 | elapsed time per iteration (s): 5.39 | learning rate: 1.622E-04 | global batch size: 512 | lm loss: 3.071618E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.914 | TFLOPs: 71.69 | +[default7]: iteration 3230/ 3814 | consumed samples: 1653760 | consumed tokens: 3386900480 | elapsed time per iteration (s): 5.38 | learning rate: 1.620E-04 | global batch size: 512 | lm loss: 3.075682E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.127 | TFLOPs: 71.85 | +[default7]: iteration 3231/ 3814 | consumed samples: 1654272 | consumed tokens: 3387949056 | elapsed time per iteration (s): 5.39 | learning rate: 1.618E-04 | global batch size: 512 | lm loss: 3.041343E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.036 | TFLOPs: 71.78 | +[default7]: iteration 3232/ 3814 | consumed samples: 1654784 | consumed tokens: 3388997632 | elapsed time per iteration (s): 5.38 | learning rate: 1.616E-04 | global batch size: 512 | lm loss: 3.068035E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.127 | TFLOPs: 71.85 | +[default7]: iteration 3233/ 3814 | consumed samples: 1655296 | consumed tokens: 3390046208 | elapsed time per iteration (s): 5.39 | learning rate: 1.614E-04 | global batch size: 512 | lm loss: 3.075380E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.061 | TFLOPs: 71.80 | +[default7]: iteration 3234/ 3814 | consumed samples: 1655808 | consumed tokens: 3391094784 | elapsed time per iteration (s): 5.38 | learning rate: 1.612E-04 | global batch size: 512 | lm loss: 3.037847E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.215 | TFLOPs: 71.92 | +[default7]: iteration 3235/ 3814 | consumed samples: 1656320 | consumed tokens: 3392143360 | elapsed time per iteration (s): 5.37 | learning rate: 1.610E-04 | global batch size: 512 | lm loss: 3.029988E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.262 | TFLOPs: 71.95 | +[default7]: iteration 3236/ 3814 | consumed samples: 1656832 | consumed tokens: 3393191936 | elapsed time per iteration (s): 5.37 | learning rate: 1.608E-04 | global batch size: 512 | lm loss: 3.051831E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.361 | TFLOPs: 72.03 | +[default7]: iteration 3237/ 3814 | consumed samples: 1657344 | consumed tokens: 3394240512 | elapsed time per iteration (s): 5.37 | learning rate: 1.606E-04 | global batch size: 512 | lm loss: 3.050937E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.278 | TFLOPs: 71.96 | +[default7]: iteration 3238/ 3814 | consumed samples: 1657856 | consumed tokens: 3395289088 | elapsed time per iteration (s): 5.38 | learning rate: 1.604E-04 | global batch size: 512 | lm loss: 3.069344E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.248 | TFLOPs: 71.94 | +[default7]: iteration 3239/ 3814 | consumed samples: 1658368 | consumed tokens: 3396337664 | elapsed time per iteration (s): 5.37 | learning rate: 1.602E-04 | global batch size: 512 | lm loss: 3.070570E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.296 | TFLOPs: 71.98 | +[default7]: iteration 3240/ 3814 | consumed samples: 1658880 | consumed tokens: 3397386240 | elapsed time per iteration (s): 5.41 | learning rate: 1.600E-04 | global batch size: 512 | lm loss: 3.069708E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.558 | TFLOPs: 71.42 | +[default7]: iteration 3241/ 3814 | consumed samples: 1659392 | consumed tokens: 3398434816 | elapsed time per iteration (s): 5.39 | learning rate: 1.598E-04 | global batch size: 512 | lm loss: 3.050078E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.988 | TFLOPs: 71.75 | +[default7]: iteration 3242/ 3814 | consumed samples: 1659904 | consumed tokens: 3399483392 | elapsed time per iteration (s): 5.41 | learning rate: 1.596E-04 | global batch size: 512 | lm loss: 3.066203E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.641 | TFLOPs: 71.48 | +[default7]: iteration 3243/ 3814 | consumed samples: 1660416 | consumed tokens: 3400531968 | elapsed time per iteration (s): 5.45 | learning rate: 1.594E-04 | global batch size: 512 | lm loss: 3.048893E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 93.993 | TFLOPs: 70.99 | +[default7]: iteration 3244/ 3814 | consumed samples: 1660928 | consumed tokens: 3401580544 | elapsed time per iteration (s): 5.43 | learning rate: 1.592E-04 | global batch size: 512 | lm loss: 3.075851E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.304 | TFLOPs: 71.23 | +[default7]: iteration 3245/ 3814 | consumed samples: 1661440 | consumed tokens: 3402629120 | elapsed time per iteration (s): 5.44 | learning rate: 1.590E-04 | global batch size: 512 | lm loss: 3.016478E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.081 | TFLOPs: 71.06 | +[default7]: iteration 3246/ 3814 | consumed samples: 1661952 | consumed tokens: 3403677696 | elapsed time per iteration (s): 5.44 | learning rate: 1.588E-04 | global batch size: 512 | lm loss: 3.055235E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.169 | TFLOPs: 71.13 | +[default7]: iteration 3247/ 3814 | consumed samples: 1662464 | consumed tokens: 3404726272 | elapsed time per iteration (s): 5.40 | learning rate: 1.586E-04 | global batch size: 512 | lm loss: 3.052098E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.760 | TFLOPs: 71.57 | +[default7]: iteration 3248/ 3814 | consumed samples: 1662976 | consumed tokens: 3405774848 | elapsed time per iteration (s): 5.42 | learning rate: 1.584E-04 | global batch size: 512 | lm loss: 3.071443E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.485 | TFLOPs: 71.37 | +[default7]: iteration 3249/ 3814 | consumed samples: 1663488 | consumed tokens: 3406823424 | elapsed time per iteration (s): 5.43 | learning rate: 1.582E-04 | global batch size: 512 | lm loss: 3.076562E+00 | loss scale: 262144.0 | grad norm: 0.121 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.338 | TFLOPs: 71.25 | +[default1]:[2023-02-16 11:13:22,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_01-model_01-model_states.pt... +[default0]:saving checkpoint at iteration 3250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default0]:[2023-02-16 11:13:22,734] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step3250 is begin to save! +[default0]:[2023-02-16 11:13:22,737] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_01-model_00-model_states.pt... +[default7]: iteration 3250/ 3814 | consumed samples: 1664000 | consumed tokens: 3407872000 | elapsed time per iteration (s): 5.37 | learning rate: 1.580E-04 | global batch size: 512 | lm loss: 3.072387E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.344 | TFLOPs: 72.01 | +[default1]:[2023-02-16 11:13:22,881] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_01-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:22,882] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_04-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:22,936] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_04-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:22,937] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_05-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:22,889] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_01-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:22,890] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_04-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:22,943] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_04-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:22,943] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_05-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:22,988] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_05-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:22,989] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_06-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,042] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_06-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,042] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_07-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:22,996] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_05-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:22,997] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_06-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,050] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_06-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,051] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_07-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,096] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_07-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,096] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_08-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,153] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_08-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,154] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_09-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,106] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_07-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,107] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_08-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,165] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_08-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,166] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_09-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_09-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,212] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_10-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,221] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_09-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,221] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_10-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_10-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,271] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_11-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,322] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_11-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,322] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_12-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,277] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_10-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,277] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_11-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,329] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_11-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,330] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_12-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_12-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,376] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_13-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,430] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_13-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,430] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_14-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,384] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_12-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,385] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_13-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,438] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_13-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,438] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_14-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,486] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_14-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,487] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_15-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,542] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_15-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,543] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_16-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,494] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_14-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,495] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_15-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,550] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_15-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,551] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_16-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,600] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_16-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,601] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_17-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,655] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_17-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,656] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_18-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,607] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_16-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,608] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_17-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,659] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_17-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,659] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_18-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,706] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_18-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,706] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_19-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,758] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_19-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,759] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_20-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,712] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_18-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,712] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_19-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,765] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_19-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,766] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_20-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,809] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_20-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,810] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_21-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,862] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_21-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,862] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_22-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,817] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_20-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,818] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_21-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,870] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_21-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,870] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_22-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:23,916] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_22-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,917] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_23-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:23,968] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_23-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:23,968] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_24-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:23,922] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_22-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,923] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_23-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:23,975] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_23-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:23,976] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_24-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,020] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_24-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,021] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_25-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,028] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_24-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,028] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_25-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,075] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_25-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,076] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_26-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,126] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_26-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,127] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_27-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,080] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_25-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,081] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_26-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:24,133] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_26-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,133] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_27-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,178] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_27-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,178] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_28-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,233] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_28-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,234] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_29-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,186] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_27-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,186] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_28-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:24,242] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_28-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,243] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_29-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,285] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_29-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,286] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_30-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_30-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,344] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_31-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,296] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_29-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,297] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_30-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:24,352] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_30-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,353] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_31-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,402] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_31-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,403] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_32-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,455] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_32-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,456] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_33-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,416] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_31-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,417] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_32-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:24,471] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_32-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,471] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_33-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,517] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_33-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,518] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_34-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,569] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_34-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,570] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_35-model_01-model_states.pt... +[default0]:[2023-02-16 11:13:24,530] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_33-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,530] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_34-model_00-model_states.pt... +[default1]:[2023-02-16 11:13:24,620] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_35-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,620] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_37-model_01-model_states.pt... +[default1]:[2023-02-16 11:13:24,621] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_37-model_01-model_states.pt. +[default1]:[2023-02-16 11:13:24,623] [INFO] [logging.py:68:log_dist] [Rank 1] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_01_model_states.pt +[default1]:[2023-02-16 11:13:24,623] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_01_model_states.pt... +[default1]:[2023-02-16 11:13:24,625] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_01_model_states.pt. +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 11:13:24,587] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_34-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,588] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_35-model_00-model_states.pt... +[default0]:[2023-02-16 11:13:24,640] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_35-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,640] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_37-model_00-model_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 11:13:24,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_01_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 11:13:24,641] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/layer_37-model_00-model_states.pt. +[default0]:[2023-02-16 11:13:24,642] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_00_model_states.pt +[default0]:[2023-02-16 11:13:24,642] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_00_model_states.pt... +[default0]:[2023-02-16 11:13:24,644] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/mp_rank_00_model_states.pt. +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 11:13:24,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_01_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_00_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_00_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_00_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_01_optim_states.pt... +[default1]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_01_optim_states.pt... +[default4]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_00_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_00_optim_states.pt... +[default2]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 11:13:24,672] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_01_optim_states.pt... +[default6]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_00_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_00_optim_states.pt... +[default5]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_01_optim_states.pt... +[default7]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_01_optim_states.pt... +[default0]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_00_optim_states.pt... +[default3]:[2023-02-16 11:13:24,671] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_01_optim_states.pt... +[default5]:[2023-02-16 11:13:25,195] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,195] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,195] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,171] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,171] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,171] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,191] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,191] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,191] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,181] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,181] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,181] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,184] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,184] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,184] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,192] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,192] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,192] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,167] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,167] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,167] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,216] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,216] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_3_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,216] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,200] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,201] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,201] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,207] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,207] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,207] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,230] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,230] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,230] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,236] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,236] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,236] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,235] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,235] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,235] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,250] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,250] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,251] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,197] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,197] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_23_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,197] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,252] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,253] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_18_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,253] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,196] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,200] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_0_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,200] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,223] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,223] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_2_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,223] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,212] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,212] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,212] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,227] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,227] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_12_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,227] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,220] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,220] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_1_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,220] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,234] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,235] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_25_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,235] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,246] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,246] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,246] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,313] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_30_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,313] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,324] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,324] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,324] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,270] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,270] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,270] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,304] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,304] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,304] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,333] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_01_optim_states.pt. +[default6]:[2023-02-16 11:13:25,243] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,244] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_15_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,244] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,326] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,326] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,326] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,292] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,292] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,293] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,319] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,320] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,283] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,283] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_20_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,283] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,335] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_5_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,335] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,332] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,332] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_11_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,332] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,345] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,346] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,346] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,286] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,286] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_27_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,286] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,321] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,322] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,322] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,276] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,276] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,276] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,276] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,276] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,363] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_00_optim_states.pt. +[default7]:[2023-02-16 11:13:25,282] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,283] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_7_mp_rank_01_optim_states.pt +[default0]:[2023-02-16 11:13:25,364] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,364] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,283] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,344] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,344] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_9_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,345] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,339] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,339] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_16_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,339] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,316] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,316] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,316] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,342] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,342] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_10_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,342] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,317] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,317] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_24_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,317] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,303] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,303] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,303] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,315] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_26_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,315] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,314] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,314] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,313] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,313] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_21_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,313] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,331] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,331] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,331] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default5]:[2023-02-16 11:13:25,314] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_01_optim_states.pt. +[default5]:[2023-02-16 11:13:25,315] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_22_mp_rank_01_optim_states.pt +[default5]:[2023-02-16 11:13:25,315] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,373] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,373] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_6_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,373] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default6]:[2023-02-16 11:13:25,335] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_00_optim_states.pt. +[default6]:[2023-02-16 11:13:25,335] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_19_mp_rank_00_optim_states.pt +[default6]:[2023-02-16 11:13:25,335] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default2]:[2023-02-16 11:13:25,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,374] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_17_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,374] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,333] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_13_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,333] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:[2023-02-16 11:13:25,415] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_01_optim_states.pt. +[default7]:[2023-02-16 11:13:25,415] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_31_mp_rank_01_optim_states.pt +[default7]:[2023-02-16 11:13:25,415] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,376] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,376] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_4_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,376] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default4]:[2023-02-16 11:13:25,346] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_00_optim_states.pt. +[default4]:[2023-02-16 11:13:25,346] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_14_mp_rank_00_optim_states.pt +[default4]:[2023-02-16 11:13:25,347] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default3]:[2023-02-16 11:13:25,374] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_01_optim_states.pt. +[default3]:[2023-02-16 11:13:25,375] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_01_optim_states.pt +[default3]:[2023-02-16 11:13:25,375] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,368] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,368] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,368] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default1]:[2023-02-16 11:13:25,356] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_01_optim_states.pt. +[default1]:[2023-02-16 11:13:25,356] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_8_mp_rank_01_optim_states.pt +[default1]:[2023-02-16 11:13:25,356] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]: successfully saved checkpoint at iteration 3250 to /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main +[default2]:[2023-02-16 11:13:25,431] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_00_optim_states.pt. +[default2]:[2023-02-16 11:13:25,431] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_29_mp_rank_00_optim_states.pt +[default2]:[2023-02-16 11:13:25,431] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default0]:[2023-02-16 11:13:25,449] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_00_optim_states.pt. +[default0]:[2023-02-16 11:13:25,449] [INFO] [engine.py:3199:_save_zero_checkpoint] zero checkpoint saved /gpfsscratch/rech/ajs/commun/checkpoints/tr14-2B7-lr0.01-init0.1-inpm10-outm10-atnm10-mup/checkpoints/main/global_step3250/zero_pp_rank_28_mp_rank_00_optim_states.pt +[default0]:[2023-02-16 11:13:25,449] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3250 is ready now! +[default7]:time (ms) | save-checkpoint: 2716.61 +[default7]: iteration 3251/ 3814 | consumed samples: 1664512 | consumed tokens: 3408920576 | elapsed time per iteration (s): 8.10 | learning rate: 1.578E-04 | global batch size: 512 | lm loss: 3.038261E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 63.244 | TFLOPs: 47.77 | +[default7]: iteration 3252/ 3814 | consumed samples: 1665024 | consumed tokens: 3409969152 | elapsed time per iteration (s): 5.40 | learning rate: 1.576E-04 | global batch size: 512 | lm loss: 3.063932E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.897 | TFLOPs: 71.68 | +[default7]: iteration 3253/ 3814 | consumed samples: 1665536 | consumed tokens: 3411017728 | elapsed time per iteration (s): 5.38 | learning rate: 1.574E-04 | global batch size: 512 | lm loss: 3.052106E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.117 | TFLOPs: 71.84 | +[default7]: iteration 3254/ 3814 | consumed samples: 1666048 | consumed tokens: 3412066304 | elapsed time per iteration (s): 5.36 | learning rate: 1.572E-04 | global batch size: 512 | lm loss: 3.057052E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.514 | TFLOPs: 72.14 | +[default7]: iteration 3255/ 3814 | consumed samples: 1666560 | consumed tokens: 3413114880 | elapsed time per iteration (s): 5.39 | learning rate: 1.570E-04 | global batch size: 512 | lm loss: 3.062044E+00 | loss scale: 262144.0 | grad norm: 0.111 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.963 | TFLOPs: 71.73 | +[default7]: iteration 3256/ 3814 | consumed samples: 1667072 | consumed tokens: 3414163456 | elapsed time per iteration (s): 5.39 | learning rate: 1.568E-04 | global batch size: 512 | lm loss: 3.075002E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.942 | TFLOPs: 71.71 | +[default7]: iteration 3257/ 3814 | consumed samples: 1667584 | consumed tokens: 3415212032 | elapsed time per iteration (s): 5.42 | learning rate: 1.566E-04 | global batch size: 512 | lm loss: 3.062772E+00 | loss scale: 262144.0 | grad norm: 0.106 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.530 | TFLOPs: 71.40 | +[default7]: iteration 3258/ 3814 | consumed samples: 1668096 | consumed tokens: 3416260608 | elapsed time per iteration (s): 5.43 | learning rate: 1.564E-04 | global batch size: 512 | lm loss: 3.009615E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.375 | TFLOPs: 71.28 | +[default7]: iteration 3259/ 3814 | consumed samples: 1668608 | consumed tokens: 3417309184 | elapsed time per iteration (s): 5.44 | learning rate: 1.562E-04 | global batch size: 512 | lm loss: 3.069433E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.168 | TFLOPs: 71.13 | +[default7]: iteration 3260/ 3814 | consumed samples: 1669120 | consumed tokens: 3418357760 | elapsed time per iteration (s): 5.44 | learning rate: 1.560E-04 | global batch size: 512 | lm loss: 3.023424E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.150 | TFLOPs: 71.11 | +[default7]: iteration 3261/ 3814 | consumed samples: 1669632 | consumed tokens: 3419406336 | elapsed time per iteration (s): 5.41 | learning rate: 1.558E-04 | global batch size: 512 | lm loss: 3.041589E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.554 | TFLOPs: 71.42 | +[default7]: iteration 3262/ 3814 | consumed samples: 1670144 | consumed tokens: 3420454912 | elapsed time per iteration (s): 5.40 | learning rate: 1.556E-04 | global batch size: 512 | lm loss: 3.099280E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.731 | TFLOPs: 71.55 | +[default7]: iteration 3263/ 3814 | consumed samples: 1670656 | consumed tokens: 3421503488 | elapsed time per iteration (s): 5.42 | learning rate: 1.554E-04 | global batch size: 512 | lm loss: 3.055191E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.529 | TFLOPs: 71.40 | +[default7]: iteration 3264/ 3814 | consumed samples: 1671168 | consumed tokens: 3422552064 | elapsed time per iteration (s): 5.43 | learning rate: 1.552E-04 | global batch size: 512 | lm loss: 3.058775E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.331 | TFLOPs: 71.25 | +[default7]: iteration 3265/ 3814 | consumed samples: 1671680 | consumed tokens: 3423600640 | elapsed time per iteration (s): 5.43 | learning rate: 1.550E-04 | global batch size: 512 | lm loss: 3.063044E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.325 | TFLOPs: 71.24 | +[default7]: iteration 3266/ 3814 | consumed samples: 1672192 | consumed tokens: 3424649216 | elapsed time per iteration (s): 5.39 | learning rate: 1.548E-04 | global batch size: 512 | lm loss: 3.015359E+00 | loss scale: 262144.0 | grad norm: 0.109 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.995 | TFLOPs: 71.75 | +[default7]: iteration 3267/ 3814 | consumed samples: 1672704 | consumed tokens: 3425697792 | elapsed time per iteration (s): 5.40 | learning rate: 1.546E-04 | global batch size: 512 | lm loss: 3.037043E+00 | loss scale: 262144.0 | grad norm: 0.110 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.781 | TFLOPs: 71.59 | +[default7]: iteration 3268/ 3814 | consumed samples: 1673216 | consumed tokens: 3426746368 | elapsed time per iteration (s): 5.41 | learning rate: 1.544E-04 | global batch size: 512 | lm loss: 3.100663E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.604 | TFLOPs: 71.46 | +[default7]: iteration 3269/ 3814 | consumed samples: 1673728 | consumed tokens: 3427794944 | elapsed time per iteration (s): 5.42 | learning rate: 1.542E-04 | global batch size: 512 | lm loss: 3.077466E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.453 | TFLOPs: 71.34 | +[default7]: iteration 3270/ 3814 | consumed samples: 1674240 | consumed tokens: 3428843520 | elapsed time per iteration (s): 5.39 | learning rate: 1.540E-04 | global batch size: 512 | lm loss: 2.992486E+00 | loss scale: 262144.0 | grad norm: 0.112 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.969 | TFLOPs: 71.73 | +[default7]: iteration 3271/ 3814 | consumed samples: 1674752 | consumed tokens: 3429892096 | elapsed time per iteration (s): 5.39 | learning rate: 1.538E-04 | global batch size: 512 | lm loss: 3.044846E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.043 | TFLOPs: 71.79 | +[default7]: iteration 3272/ 3814 | consumed samples: 1675264 | consumed tokens: 3430940672 | elapsed time per iteration (s): 5.37 | learning rate: 1.536E-04 | global batch size: 512 | lm loss: 3.034719E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.268 | TFLOPs: 71.96 | +[default7]: iteration 3273/ 3814 | consumed samples: 1675776 | consumed tokens: 3431989248 | elapsed time per iteration (s): 5.36 | learning rate: 1.534E-04 | global batch size: 512 | lm loss: 3.058285E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.490 | TFLOPs: 72.12 | +[default7]: iteration 3274/ 3814 | consumed samples: 1676288 | consumed tokens: 3433037824 | elapsed time per iteration (s): 5.38 | learning rate: 1.532E-04 | global batch size: 512 | lm loss: 3.053731E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.198 | TFLOPs: 71.90 | +[default7]: iteration 3275/ 3814 | consumed samples: 1676800 | consumed tokens: 3434086400 | elapsed time per iteration (s): 5.39 | learning rate: 1.530E-04 | global batch size: 512 | lm loss: 3.049986E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.022 | TFLOPs: 71.77 | +[default7]: iteration 3276/ 3814 | consumed samples: 1677312 | consumed tokens: 3435134976 | elapsed time per iteration (s): 5.38 | learning rate: 1.529E-04 | global batch size: 512 | lm loss: 3.063300E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.094 | TFLOPs: 71.83 | +[default7]: iteration 3277/ 3814 | consumed samples: 1677824 | consumed tokens: 3436183552 | elapsed time per iteration (s): 5.37 | learning rate: 1.527E-04 | global batch size: 512 | lm loss: 3.053318E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.340 | TFLOPs: 72.01 | +[default7]: iteration 3278/ 3814 | consumed samples: 1678336 | consumed tokens: 3437232128 | elapsed time per iteration (s): 5.37 | learning rate: 1.525E-04 | global batch size: 512 | lm loss: 3.036964E+00 | loss scale: 262144.0 | grad norm: 0.123 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.359 | TFLOPs: 72.03 | +[default7]: iteration 3279/ 3814 | consumed samples: 1678848 | consumed tokens: 3438280704 | elapsed time per iteration (s): 5.38 | learning rate: 1.523E-04 | global batch size: 512 | lm loss: 3.070071E+00 | loss scale: 262144.0 | grad norm: 0.119 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.132 | TFLOPs: 71.85 | +[default7]: iteration 3280/ 3814 | consumed samples: 1679360 | consumed tokens: 3439329280 | elapsed time per iteration (s): 5.39 | learning rate: 1.521E-04 | global batch size: 512 | lm loss: 3.056597E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.046 | TFLOPs: 71.79 | +[default7]: iteration 3281/ 3814 | consumed samples: 1679872 | consumed tokens: 3440377856 | elapsed time per iteration (s): 5.38 | learning rate: 1.519E-04 | global batch size: 512 | lm loss: 3.018635E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.130 | TFLOPs: 71.85 | +[default7]: iteration 3282/ 3814 | consumed samples: 1680384 | consumed tokens: 3441426432 | elapsed time per iteration (s): 5.38 | learning rate: 1.517E-04 | global batch size: 512 | lm loss: 3.048617E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.157 | TFLOPs: 71.87 | +[default7]: iteration 3283/ 3814 | consumed samples: 1680896 | consumed tokens: 3442475008 | elapsed time per iteration (s): 5.39 | learning rate: 1.515E-04 | global batch size: 512 | lm loss: 3.038512E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.029 | TFLOPs: 71.78 | +[default7]: iteration 3284/ 3814 | consumed samples: 1681408 | consumed tokens: 3443523584 | elapsed time per iteration (s): 5.37 | learning rate: 1.513E-04 | global batch size: 512 | lm loss: 3.052964E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.366 | TFLOPs: 72.03 | +[default7]: iteration 3285/ 3814 | consumed samples: 1681920 | consumed tokens: 3444572160 | elapsed time per iteration (s): 5.40 | learning rate: 1.511E-04 | global batch size: 512 | lm loss: 3.032352E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.837 | TFLOPs: 71.63 | +[default7]: iteration 3286/ 3814 | consumed samples: 1682432 | consumed tokens: 3445620736 | elapsed time per iteration (s): 5.40 | learning rate: 1.509E-04 | global batch size: 512 | lm loss: 3.099761E+00 | loss scale: 262144.0 | grad norm: 0.122 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.755 | TFLOPs: 71.57 | +[default7]: iteration 3287/ 3814 | consumed samples: 1682944 | consumed tokens: 3446669312 | elapsed time per iteration (s): 5.37 | learning rate: 1.508E-04 | global batch size: 512 | lm loss: 3.023891E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.325 | TFLOPs: 72.00 | +[default7]: iteration 3288/ 3814 | consumed samples: 1683456 | consumed tokens: 3447717888 | elapsed time per iteration (s): 5.38 | learning rate: 1.506E-04 | global batch size: 512 | lm loss: 3.015826E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.105 | TFLOPs: 71.83 | +[default7]: iteration 3289/ 3814 | consumed samples: 1683968 | consumed tokens: 3448766464 | elapsed time per iteration (s): 5.39 | learning rate: 1.504E-04 | global batch size: 512 | lm loss: 3.067842E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.930 | TFLOPs: 71.70 | +[default7]: iteration 3290/ 3814 | consumed samples: 1684480 | consumed tokens: 3449815040 | elapsed time per iteration (s): 5.38 | learning rate: 1.502E-04 | global batch size: 512 | lm loss: 3.041881E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.177 | TFLOPs: 71.89 | +[default7]: iteration 3291/ 3814 | consumed samples: 1684992 | consumed tokens: 3450863616 | elapsed time per iteration (s): 5.37 | learning rate: 1.500E-04 | global batch size: 512 | lm loss: 3.052777E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.256 | TFLOPs: 71.95 | +[default7]: iteration 3292/ 3814 | consumed samples: 1685504 | consumed tokens: 3451912192 | elapsed time per iteration (s): 5.40 | learning rate: 1.498E-04 | global batch size: 512 | lm loss: 3.051323E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.793 | TFLOPs: 71.60 | +[default7]: iteration 3293/ 3814 | consumed samples: 1686016 | consumed tokens: 3452960768 | elapsed time per iteration (s): 5.41 | learning rate: 1.496E-04 | global batch size: 512 | lm loss: 3.051794E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.577 | TFLOPs: 71.44 | +[default7]: iteration 3294/ 3814 | consumed samples: 1686528 | consumed tokens: 3454009344 | elapsed time per iteration (s): 5.41 | learning rate: 1.494E-04 | global batch size: 512 | lm loss: 3.040294E+00 | loss scale: 262144.0 | grad norm: 0.120 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.601 | TFLOPs: 71.45 | +[default7]: iteration 3295/ 3814 | consumed samples: 1687040 | consumed tokens: 3455057920 | elapsed time per iteration (s): 5.40 | learning rate: 1.493E-04 | global batch size: 512 | lm loss: 3.066223E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.742 | TFLOPs: 71.56 | +[default7]: iteration 3296/ 3814 | consumed samples: 1687552 | consumed tokens: 3456106496 | elapsed time per iteration (s): 5.43 | learning rate: 1.491E-04 | global batch size: 512 | lm loss: 3.077192E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.308 | TFLOPs: 71.23 | +[default7]: iteration 3297/ 3814 | consumed samples: 1688064 | consumed tokens: 3457155072 | elapsed time per iteration (s): 5.40 | learning rate: 1.489E-04 | global batch size: 512 | lm loss: 3.043060E+00 | loss scale: 262144.0 | grad norm: 0.113 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.776 | TFLOPs: 71.59 | +[default7]: iteration 3298/ 3814 | consumed samples: 1688576 | consumed tokens: 3458203648 | elapsed time per iteration (s): 5.38 | learning rate: 1.487E-04 | global batch size: 512 | lm loss: 3.059057E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.216 | TFLOPs: 71.92 | +[default7]: iteration 3299/ 3814 | consumed samples: 1689088 | consumed tokens: 3459252224 | elapsed time per iteration (s): 5.41 | learning rate: 1.485E-04 | global batch size: 512 | lm loss: 3.036532E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.553 | TFLOPs: 71.42 | +[default7]: iteration 3300/ 3814 | consumed samples: 1689600 | consumed tokens: 3460300800 | elapsed time per iteration (s): 5.42 | learning rate: 1.483E-04 | global batch size: 512 | lm loss: 3.033228E+00 | loss scale: 262144.0 | grad norm: 0.127 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.406 | TFLOPs: 71.31 | +[default7]: iteration 3301/ 3814 | consumed samples: 1690112 | consumed tokens: 3461349376 | elapsed time per iteration (s): 5.38 | learning rate: 1.481E-04 | global batch size: 512 | lm loss: 3.097452E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.199 | TFLOPs: 71.90 | +[default7]: iteration 3302/ 3814 | consumed samples: 1690624 | consumed tokens: 3462397952 | elapsed time per iteration (s): 5.40 | learning rate: 1.480E-04 | global batch size: 512 | lm loss: 3.028832E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.884 | TFLOPs: 71.67 | +[default7]: iteration 3303/ 3814 | consumed samples: 1691136 | consumed tokens: 3463446528 | elapsed time per iteration (s): 5.38 | learning rate: 1.478E-04 | global batch size: 512 | lm loss: 3.034434E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.193 | TFLOPs: 71.90 | +[default7]: iteration 3304/ 3814 | consumed samples: 1691648 | consumed tokens: 3464495104 | elapsed time per iteration (s): 5.40 | learning rate: 1.476E-04 | global batch size: 512 | lm loss: 3.075241E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.893 | TFLOPs: 71.67 | +[default7]: iteration 3305/ 3814 | consumed samples: 1692160 | consumed tokens: 3465543680 | elapsed time per iteration (s): 5.40 | learning rate: 1.474E-04 | global batch size: 512 | lm loss: 3.035881E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.812 | TFLOPs: 71.61 | +[default7]: iteration 3306/ 3814 | consumed samples: 1692672 | consumed tokens: 3466592256 | elapsed time per iteration (s): 5.40 | learning rate: 1.472E-04 | global batch size: 512 | lm loss: 3.067844E+00 | loss scale: 262144.0 | grad norm: 0.118 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.827 | TFLOPs: 71.62 | +[default7]: iteration 3307/ 3814 | consumed samples: 1693184 | consumed tokens: 3467640832 | elapsed time per iteration (s): 5.41 | learning rate: 1.470E-04 | global batch size: 512 | lm loss: 3.033460E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.661 | TFLOPs: 71.50 | +[default7]: iteration 3308/ 3814 | consumed samples: 1693696 | consumed tokens: 3468689408 | elapsed time per iteration (s): 5.39 | learning rate: 1.469E-04 | global batch size: 512 | lm loss: 3.045043E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.060 | TFLOPs: 71.80 | +[default7]: iteration 3309/ 3814 | consumed samples: 1694208 | consumed tokens: 3469737984 | elapsed time per iteration (s): 5.41 | learning rate: 1.467E-04 | global batch size: 512 | lm loss: 3.070357E+00 | loss scale: 262144.0 | grad norm: 0.114 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.671 | TFLOPs: 71.51 | +[default7]: iteration 3310/ 3814 | consumed samples: 1694720 | consumed tokens: 3470786560 | elapsed time per iteration (s): 5.39 | learning rate: 1.465E-04 | global batch size: 512 | lm loss: 3.009707E+00 | loss scale: 262144.0 | grad norm: 0.115 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 94.918 | TFLOPs: 71.69 | +[default7]: iteration 3311/ 3814 | consumed samples: 1695232 | consumed tokens: 3471835136 | elapsed time per iteration (s): 5.38 | learning rate: 1.463E-04 | global batch size: 512 | lm loss: 3.021866E+00 | loss scale: 262144.0 | grad norm: 0.117 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.095 | TFLOPs: 71.83 | +[default7]: iteration 3312/ 3814 | consumed samples: 1695744 | consumed tokens: 3472883712 | elapsed time per iteration (s): 5.39 | learning rate: 1.461E-04 | global batch size: 512 | lm loss: 3.067137E+00 | loss scale: 262144.0 | grad norm: 0.116 | num zeros: 0.0 | number of skipped iterations: 0 | number of nan iterations: 0 | samples per second: 95.067 | TFLOPs: 71.80 | +slurmstepd: error: *** STEP 1219018.0 ON jean-zay-iam09 CANCELLED AT 2023-02-16T11:19:04 DUE TO TIME LIMIT *** +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99252 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114429 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99253 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114430 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925389 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99254 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925390 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99940 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114431 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925391 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114432 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99255 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844061 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99941 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114433 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925392 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106863 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99942 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114434 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844062 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99256 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925393 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114435 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668151 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106864 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114436 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844063 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99943 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99257 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925394 closing signal SIGTERM +WARNING:torch.distributed.elastic.agent.server.api:Received 15 death signal, shutting down workers +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99944 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442008 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99945 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99258 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925395 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99946 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668152 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106865 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442009 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925396 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99259 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99947 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844064 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668153 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844065 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106866 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844066 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442010 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106867 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442011 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668154 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106868 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106869 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844067 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106870 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668155 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844068 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668156 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442012 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668157 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442013 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668158 closing signal SIGTERM +srun: Job step aborted: Waiting up to 62 seconds for job step to finish. +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442014 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442015 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99252 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99253 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99254 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99255 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99256 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114429 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99257 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114430 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99258 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99259 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114431 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114432 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114433 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114434 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114435 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 114436 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925389 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925390 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925391 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925392 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925393 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925394 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925395 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 3925396 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99940 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99941 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99942 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99943 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99944 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99945 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99946 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 99947 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106863 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106864 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106865 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106866 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106867 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106868 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106869 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 106870 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668151 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668152 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668153 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668154 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668155 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668156 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668157 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1668158 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442008 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442009 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844061 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442010 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442011 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844062 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442012 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844063 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442013 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442014 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844064 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1442015 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844065 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844066 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844067 closing signal SIGTERM +WARNING:torch.distributed.elastic.multiprocessing.api:Sending process 1844068 closing signal SIGTERM