Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
@@ -62,6 +62,7 @@ login(token=os.environ["HF_ACCESS_TOKEN"])
|
|
62 |
base_model = "EleutherAI/gpt-neo-1.3B" #load_8bit = False (in load_tokenizer_and_model)
|
63 |
#base_model = "TheBloke/airoboros-13B-HF" #load_8bit = True
|
64 |
tokenizer,model,device = load_tokenizer_and_model(base_model, False)
|
|
|
65 |
|
66 |
dataset_neu = daten_laden("alexkueck/tis")
|
67 |
#dataset_neu = daten_laden("EleutherAI/pile")
|
|
|
62 |
base_model = "EleutherAI/gpt-neo-1.3B" #load_8bit = False (in load_tokenizer_and_model)
|
63 |
#base_model = "TheBloke/airoboros-13B-HF" #load_8bit = True
|
64 |
tokenizer,model,device = load_tokenizer_and_model(base_model, False)
|
65 |
+
tokenizer.add_special_tokens({'pad_token': '[PAD]'})
|
66 |
|
67 |
dataset_neu = daten_laden("alexkueck/tis")
|
68 |
#dataset_neu = daten_laden("EleutherAI/pile")
|