Namespace(align_suffix=None, alignfile=None, all_gather_list_size=16384, azureml_logging=False, bf16=False, bpe=None, cpu=False, criterion='cross_entropy', dataset_impl='mmap', destdir='data-bin/man-2-hanzi-bpe10k', empty_cache_freq=0, fp16=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=None, joined_dictionary=True, log_format=None, log_interval=100, lr_scheduler='fixed', memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, model_parallel_size=1, no_progress_bar=False, nwordssrc=-1, nwordstgt=-1, only_source=False, optimizer=None, padding_factor=8, plasma_path='/tmp/plasma', profile=False, quantization_config_path=None, reset_logging=False, scoring='bleu', seed=1, simul_type=None, source_lang='man', srcdict=None, suppress_crashes=False, target_lang='hanzi', task='translation', tensorboard_logdir=None, testpref='YYTD/eval/eval.10000.bpe', tgtdict=None, threshold_loss_scale=None, thresholdsrc=0, thresholdtgt=0, tokenizer=None, tpu=False, trainpref='YYTD/train.10000.bpe', use_plasma_view=False, user_dir=None, validpref='YYTD/201214/eval/eval.10000.bpe', wandb_project=None, workers=20)
[man] Dictionary: 23080 types
[man] YYTD/train.10000.bpe.man: 3227095 sents, 20701981 tokens, 0.0% replaced by <unk>
[man] Dictionary: 23080 types
[man] YYTD/201214/eval/eval.10000.bpe.man: 6212 sents, 37893 tokens, 0.00264% replaced by <unk>
[man] Dictionary: 23080 types
[man] YYTD/eval/eval.10000.bpe.man: 6212 sents, 37893 tokens, 0.00264% replaced by <unk>
[hanzi] Dictionary: 23080 types
[hanzi] YYTD/train.10000.bpe.hanzi: 3227095 sents, 20946168 tokens, 0.0% replaced by <unk>
[hanzi] Dictionary: 23080 types
[hanzi] YYTD/201214/eval/eval.10000.bpe.hanzi: 6212 sents, 43206 tokens, 0.0% replaced by <unk>
[hanzi] Dictionary: 23080 types
[hanzi] YYTD/eval/eval.10000.bpe.hanzi: 6212 sents, 43206 tokens, 0.0% replaced by <unk>
Wrote preprocessed data to data-bin/man-2-hanzi-bpe10k