Spaces:
Runtime error
Runtime error
import json | |
from typing import List, Dict, Tuple, Union, Any | |
import streamlit as st | |
from annotated_text import annotated_text | |
sidebar = st.sidebar | |
def bs_unigram_match_annotated_text(belief_state_example) -> List[Union[str, Tuple]]: | |
gold_set = set(belief_state_example['gold'].split(' ')) | |
input_set = set(" ".join(belief_state_example['input']).split(' ')) | |
generated = belief_state_example['generated'] | |
result = [] | |
for word in generated.split(' '): | |
if word in gold_set: | |
result.append((word, 'gold', '#dfd')) # gold overlap => label green | |
elif word in input_set: | |
result.append((word, 'in', '#eea')) # input overlap => label yellow | |
else: | |
result.append(word + ' ') # no overlap => no label (replace space) | |
return result | |
# load in data | |
pptod_examples: List[Dict] = [] | |
models: Dict[str, Dict[str, Any]] = { | |
'pptod-small': { | |
'name': 'pptod-small', | |
'description': 'a T5 model that has been pre-trained on the ToD-BERT dataset **in this data format.** As such, ' | |
'it is familiar with the meaning of these special separator tokens. However, it does not have' | |
'MultiWoZ training experience, so while it has adapted to the belief state grammar generally, it ' | |
'is unaware of the particular slot name conventions of MultiWoZ.', | |
'output_file': './output/pptod-small-10-percent.jsonl' | |
}, | |
't5-small': { | |
'name': 't5-small', | |
'description': 'a T5 model with no dialogue experience. Data input has been transformed to exclude special tokens' | |
'that the model could not be familiar with.', | |
'output_file': './output/t5-small-10-percent.jsonl' | |
}, | |
'bart': { | |
'name': 'bart', | |
'description': 'a BART model with no dialogue experience. Data input has been transformed to exclude special tokens' | |
'that the model could not be familiar with.', | |
'output_file': './output/bart-100ish-examples.jsonl' | |
}, | |
'dialogpt': { | |
'name': 'dialogpt', | |
'description': 'DialoGPT is a (fine-tuned GPT-2) dialogue response generation model for multiturn conversations from 147M Reddit Conversation chains', | |
'output_file': './output/dialogpt-100ish-examples.jsonl' | |
}, | |
'my-t5-pptod-checkpoint': { | |
'name': 'my-t5-pptod-checkpoint', | |
'description': 'My re-implementation of the pptod approach, in re-structured code', | |
'output_file': './output/my-pre-train-t5-100ish-examples.jsonl' | |
}, | |
'my-t5-fine-tune-checkpoint': { | |
'name': 'my-t5-fine-tune-checkpoint', | |
'description': 'My re-implementation of the pptod approach, in re-structured code, fine-tuned on MultiWoZ', | |
'output_file': './output/my-fine-tune-t5-100ish-examples.jsonl' | |
} | |
} | |
for model_def in models.values(): | |
model_def['examples'] = [] | |
with open(model_def['output_file'], 'r') as f: | |
for line in f.readlines(): | |
model_def['examples'].append(json.loads(line.strip())) | |
model_names = list(models.keys()) | |
model_name = sidebar.selectbox('Model', model_names) | |
active_model = models[model_name] | |
st.write(f""" | |
#### Inputs | |
**Selected Model:** `{active_model['name']}` | |
{active_model['description']} | |
""") | |
""" | |
### Belief State Prediction | |
Below is the predicted belief state as a sequence. | |
- `input` denotes the input, which has been transformed into a list for | |
human readability but is presented to the model as a sequence. | |
- `gold` is the target belief state in sequence form (slot-name slot-value pairs) | |
- `generated` is the model generated belief state sequence | |
""" | |
titles = [f"{i}: {e[0]['turn_domain'][0]} (Turn {e[0]['turn_num']})" for i, e in enumerate(active_model['examples'])] | |
title = sidebar.selectbox("Development Example", titles) | |
active_example = active_model['examples'][int(title[0])][0] | |
active_belief_spans = active_example['bspn_input'].split("> <") | |
active_example_bs = {'input': | |
[ ('<' if i > 0 else '') + | |
string + | |
('>' if string[-1] is not '>' and len(active_belief_spans) > 1 else '') | |
for i, string in enumerate(active_belief_spans)], | |
'generated': active_example['bspn_gen'], | |
'gold': active_example['bspn']} | |
st.write(active_example_bs) | |
""" | |
##### Generated Overlap | |
""" | |
annotated_text(*bs_unigram_match_annotated_text(active_example_bs)) | |
""" | |
--- | |
### Response Generation | |
Below is the predicted response as a sequence. | |
- `input` denotes the input, which has been transformed into a list for | |
human readability but is presented to the model as a sequence. | |
- `gold` is the target response sequence | |
- `generated` is the model generated response | |
""" | |
#title = st.selectbox("Development Example", titles) | |
active_example_resp = {'input': | |
[ ('<' if i > 0 else '') + | |
string + | |
('>' if string[-1] is not '>' else '') | |
for i, string in enumerate(active_example['resp_input'].split("> <"))], | |
'generated': active_example['resp_gen'], | |
'gold': active_example['resp']} | |
st.write(active_example_resp) | |
""" | |
##### Generated Overlap | |
""" | |
annotated_text(*bs_unigram_match_annotated_text(active_example_resp)) | |