Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
@@ -40,6 +40,12 @@ def group_texts(examples):
|
|
40 |
result["labels"] = result["input_ids"].copy()
|
41 |
return result
|
42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
|
44 |
|
45 |
|
@@ -144,7 +150,7 @@ trainer = Trainer(
|
|
144 |
)
|
145 |
print ("trainer")
|
146 |
#trainer ausführen
|
147 |
-
|
148 |
#Wenn man vom letzten checkpoint aus weiter trainieren möchte: trainer.train(resume_from_checkpoint=True)
|
149 |
print("trained!!!!!")
|
150 |
|
@@ -161,6 +167,15 @@ print("Done Eval")
|
|
161 |
print("Test")
|
162 |
text = "Was ist Tis?"
|
163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
164 |
#encoding = tokenizer(text, return_tensors="pt")
|
165 |
#encoding = {k: v.to(trainer.model.device) for k,v in encoding.items()}
|
166 |
|
@@ -172,9 +187,12 @@ text = "Was ist Tis?"
|
|
172 |
|
173 |
print("Output:\n" )
|
174 |
#print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
|
|
175 |
|
176 |
|
177 |
-
|
|
|
|
|
178 |
#login(token=os.environ["HF_WRITE_TOKEN"])
|
179 |
#trainer.save_model("alexkueck/test-tis-1")
|
180 |
print("done")
|
|
|
40 |
result["labels"] = result["input_ids"].copy()
|
41 |
return result
|
42 |
|
43 |
+
#Generate Response - nach dem training testen, wie es funktioniert
|
44 |
+
def generate_response(prompt):
|
45 |
+
input_ids = tokenizer.encode(prompt, return_tensors="pt")
|
46 |
+
output = model.generate(input_ids, max_length=100)
|
47 |
+
response = tokenizer.decode(output[:, input_ids.shape[-1]:][0], skip_special_tokens=True)
|
48 |
+
return response
|
49 |
|
50 |
|
51 |
|
|
|
150 |
)
|
151 |
print ("trainer")
|
152 |
#trainer ausführen
|
153 |
+
trainer.train()
|
154 |
#Wenn man vom letzten checkpoint aus weiter trainieren möchte: trainer.train(resume_from_checkpoint=True)
|
155 |
print("trained!!!!!")
|
156 |
|
|
|
167 |
print("Test")
|
168 |
text = "Was ist Tis?"
|
169 |
|
170 |
+
# Example usage
|
171 |
+
prompt = "Was ist TIS?"
|
172 |
+
|
173 |
+
response = generate_response(prompt)
|
174 |
+
print(response)
|
175 |
+
print("response done")
|
176 |
+
|
177 |
+
|
178 |
+
'''
|
179 |
#encoding = tokenizer(text, return_tensors="pt")
|
180 |
#encoding = {k: v.to(trainer.model.device) for k,v in encoding.items()}
|
181 |
|
|
|
187 |
|
188 |
print("Output:\n" )
|
189 |
#print(tokenizer.decode(outputs[0], skip_special_tokens=True))
|
190 |
+
'''
|
191 |
|
192 |
|
193 |
+
###################################################
|
194 |
+
#Save to a place -????? Where????
|
195 |
+
print("Save to ???")
|
196 |
#login(token=os.environ["HF_WRITE_TOKEN"])
|
197 |
#trainer.save_model("alexkueck/test-tis-1")
|
198 |
print("done")
|