alexkueck commited on
Commit
ee00be9
·
1 Parent(s): eb5d4e5

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -3
app.py CHANGED
@@ -198,7 +198,7 @@ lm_datasets = tokenized_datasets.map(
198
 
199
  # Batches von Daten zusammenfassen
200
  tokenizer.pad_token = tokenizer.eos_token
201
- data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
202
 
203
 
204
  print ("###############lm datasets####################")
@@ -264,8 +264,8 @@ trainer = Trainer(
264
  args=training_args,
265
  train_dataset=lm_datasets["train"],
266
  eval_dataset=lm_datasets["test"],
267
- data_collator=data_collator,
268
- #tokenizer=tokenizer,
269
  compute_metrics=compute_metrics,
270
  )
271
 
 
198
 
199
  # Batches von Daten zusammenfassen
200
  tokenizer.pad_token = tokenizer.eos_token
201
+ #data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
202
 
203
 
204
  print ("###############lm datasets####################")
 
264
  args=training_args,
265
  train_dataset=lm_datasets["train"],
266
  eval_dataset=lm_datasets["test"],
267
+ #data_collator=data_collator,
268
+ tokenizer=tokenizer,
269
  compute_metrics=compute_metrics,
270
  )
271