alexkueck commited on
Commit
37000a6
·
1 Parent(s): f7dcf63

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -0
app.py CHANGED
@@ -62,6 +62,7 @@ login(token=os.environ["HF_ACCESS_TOKEN"])
62
  base_model = "EleutherAI/gpt-neo-1.3B" #load_8bit = False (in load_tokenizer_and_model)
63
  #base_model = "TheBloke/airoboros-13B-HF" #load_8bit = True
64
  tokenizer,model,device = load_tokenizer_and_model(base_model, False)
 
65
 
66
  dataset_neu = daten_laden("alexkueck/tis")
67
  #dataset_neu = daten_laden("EleutherAI/pile")
 
62
  base_model = "EleutherAI/gpt-neo-1.3B" #load_8bit = False (in load_tokenizer_and_model)
63
  #base_model = "TheBloke/airoboros-13B-HF" #load_8bit = True
64
  tokenizer,model,device = load_tokenizer_and_model(base_model, False)
65
+ tokenizer.add_special_tokens({'pad_token': '[PAD]'})
66
 
67
  dataset_neu = daten_laden("alexkueck/tis")
68
  #dataset_neu = daten_laden("EleutherAI/pile")