|
--- |
|
{} |
|
--- |
|
``` |
|
pre_text = "The following is an interaction between a user and an AI assistant that is related to the above text." |
|
def ds_map_fn(row): |
|
input = f"[[[Title]]] {row['title'].strip()}\n[[[Content]]] {row['context'].strip()}\n\n" + pre_text + "\n\n[[[User]]] " |
|
output = f"{row['question'].strip()}\n[[[Assistant]]] {row['answer'].strip()}" |
|
|
|
input = tokenizer.encode(input, add_special_tokens=False) |
|
output = tokenizer.encode(output, add_special_tokens=False) |
|
|
|
input_ids = input + output + [tokenizer.eos_token_id] |
|
labels = [-100]*len(input) + output + [tokenizer.eos_token_id] |
|
|
|
return {'input_ids': input_ids, 'labels': labels} |
|
ds = ds.map(ds_map_fn, remove_columns=ds.column_names) |
|
``` |
