Spaces:
Runtime error
Runtime error
MelikeDulkadir
commited on
Commit
•
5108918
1
Parent(s):
de1ce9e
Update app.py
Browse files
app.py
CHANGED
@@ -16,4 +16,7 @@ tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
|
|
16 |
def preprocess_function(examples):
|
17 |
return tokenizer(examples["text"], truncation=True)
|
18 |
|
19 |
-
tokenized_imdb = imdb.map(preprocess_function, batched=True)
|
|
|
|
|
|
|
|
16 |
def preprocess_function(examples):
|
17 |
return tokenizer(examples["text"], truncation=True)
|
18 |
|
19 |
+
tokenized_imdb = imdb.map(preprocess_function, batched=True)
|
20 |
+
|
21 |
+
from transformers import DataCollatorWithPadding
|
22 |
+
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
|