init

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+index/en-es_metadata_ref.pkl filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,900 @@
+import gradio as gr
+from time import time
+
+from bertviz import model_view, head_view
+from bertviz_gradio import head_view_mod
+
+import faiss
+import torch
+import os
+# import nltk
+import argparse
+import random
+import numpy as np
+import pandas as pd
+
+from argparse import Namespace
+from tqdm.notebook import tqdm
+from torch.utils.data import DataLoader
+from functools import partial
+
+from transformers import AutoTokenizer, MarianTokenizer, AutoModel, AutoModelForSeq2SeqLM, MarianMTModel
+
+model_es = "Helsinki-NLP/opus-mt-en-es"
+model_fr = "Helsinki-NLP/opus-mt-en-fr"
+model_zh = "Helsinki-NLP/opus-mt-en-zh"
+model_sw = "Helsinki-NLP/opus-mt-en-sw"
+
+tokenizer_es = AutoTokenizer.from_pretrained(model_es)
+tokenizer_fr = AutoTokenizer.from_pretrained(model_fr)
+tokenizer_zh = AutoTokenizer.from_pretrained(model_zh)
+tokenizer_sw = AutoTokenizer.from_pretrained(model_sw)
+
+model_tr_es = MarianMTModel.from_pretrained(model_es)
+model_tr_fr = MarianMTModel.from_pretrained(model_fr)
+model_tr_zh = MarianMTModel.from_pretrained(model_zh)
+model_tr_sw = MarianMTModel.from_pretrained(model_sw)
+
+from faiss import write_index, read_index
+import pickle 
+
+def load_index(model): 
+	with open('index/'+ model + '_metadata_ref.pkl', 'rb') as f:
+		loaded_dict = pickle.load(f)
+	for type in ['tokens','words']:
+		for kind in ['input', 'output']: 
+			## save index file
+			name = 'index/'+ model + "_" + kind + "_"+ type + ".index" 
+			loaded_dict[kind][type][1] = read_index(name)
+			# write_index(metadata_all[kind][type][1], name)
+	return loaded_dict
+
+
+dict_models = {
+	'en-es': model_es,
+	'en-fr': model_fr,
+	'en-zh': model_zh,
+	'en-sw': model_sw,
+}
+
+dict_models_tr = {
+	'en-es': model_tr_es,
+	'en-fr': model_tr_fr,
+	'en-zh': model_tr_zh,
+	'en-sw': model_tr_sw,
+}
+
+dict_tokenizer_tr = {
+	'en-es': tokenizer_es,
+	'en-fr': tokenizer_fr,
+	'en-zh': tokenizer_zh,
+	'en-sw': tokenizer_sw,
+}
+
+dict_reference_faiss = {
+	'en-es': load_index('en-es'),
+}
+
+saliency_examples = [
+	"Peace of Mind: Protection for consumers.",
+	"The sustainable development goals report: towards a rescue plan for people and planet",
+	"We will leave no stone unturned to hold those responsible to account.",
+	"The clock is now ticking on our work to finalise the remaining key legislative proposals presented by this Commission to ensure that citizens and businesses can reap the benefits of our policy actions.",
+	"Pumpkins, squash and gourds, fresh or chilled, excluding courgettes",
+	"The labour market participation of mothers with infants has even deteriorated over the past two decades, often impacting their career and incomes for years.",
+]
+
+contrastive_examples = [
+["Peace of Mind: Protection for consumers.",
+"Paz mental: protección de los consumidores",
+"Paz de la mente: protección de los consumidores"],
+["the slaughterer has finished his work.",
+"l'abatteur a terminé son travail.",
+"l'abatteuse a terminé son travail."],
+['A fundamental shift is needed - in commitment, solidarity, financing and action - to put the world on a better path.',
+ '需要在承诺、团结、筹资和行动方面进行根本转变,使世界走上更美好的道路。',
+ '我们需要从根本上转变承诺、团结、资助和行动，使世界走上更美好的道路。',]
+	]
+
+
+
+#Load challenge set examples
+df_challenge_set = pd.read_csv("challenge_sets.csv")
+arr_challenge_set = df_challenge_set.values
+arr_challenge_set = [[x[2], x[3], x[4], x[5]] for x in arr_challenge_set]
+
+
+
+
+def get_k_prob_tokens(transition_scores, result, model, k_values=5):
+	tokenizer_tr = dict_tokenizer_tr[model]
+	gen_sequences = result.sequences[:, 1:]
+
+	result_output = []
+
+	# First beam only... 
+	bs = 0
+	text = ' '
+	for tok, score, i_step in zip(gen_sequences[bs], transition_scores[bs],range(len(gen_sequences[bs]))):
+
+		beam_i = result.beam_indices[0][i_step]
+		if beam_i < 0:
+			beam_i = bs
+		bs_alt = [tokenizer_tr.decode(tok) for tok in result.scores[i_step][beam_i].topk(k_values).indices ]
+		bs_alt_scores = np.exp(result.scores[i_step][beam_i].topk(k_values).values)
+		result_output.append([np.array(result.scores[i_step][beam_i].topk(k_values).indices), np.array(bs_alt_scores),bs_alt])
+
+	return result_output
+
+
+def split_token_from_sequences(sequences, model) -> dict :
+	n_sentences = len(sequences)
+
+	gen_sequences_texts = []
+	for bs in range(n_sentences): 
+		# gen_sequences_texts.append(dict_tokenizer_tr[model].decode(sequences[:, 1:][bs],  skip_special_tokens=True).split(' '))
+		#### decoder per token.
+		seq_bs = []
+		
+		for token in sequences[:, 1:][bs]:
+			seq_bs.append(dict_tokenizer_tr[model].decode(token,  skip_special_tokens=True))
+		gen_sequences_texts.append(seq_bs)
+
+	score = 0
+	#raw dict is bos
+	text = 'bos'
+	new_id = text +'--1'
+	dict_parent = [{'id': new_id, 'parentId': None , 'text': text, 'name': 'bos', 'prob': score }]
+	id_dict_pos = {}
+	step_i = 0
+	cont = True
+	words_by_step = [] #[['bos' for i in range(n_sentences)]]
+
+	while cont: 
+		# append to dict_parent for all beams of step_i
+		cont = False
+		step_words = []
+		for beam in range(n_sentences):
+			app_text = '<empty_word>'
+			if step_i < len(gen_sequences_texts[beam]): 
+				app_text = gen_sequences_texts[beam][step_i]
+				cont = True
+			step_words.append(app_text)
+		words_by_step.append(step_words)
+		print(words_by_step)
+
+		for i_bs, step_w in enumerate(step_words):
+			if not step_w in ['<empty_word>', '<pad>']:
+				#new id if the same word is not in another beam (?) [beam[i] was a token id]
+				#parent id = previous word and previous step.
+
+				
+				# new_parent_id = "-".join([str(beam[i]) for i in range(step_i)])
+				
+				new_id = "-".join([str(words_by_step[i][i_bs])+ '-' + str(i) for i in range(step_i+1)])
+				parent_id = "-".join([words_by_step[i][i_bs] + '-' + str(i) for i in range(step_i) ])
+				
+				# new_id = step_w +'-' + str(step_i)
+				# parent_id = words_by_step[step_i-1][i_bs] + '-' + str(step_i -1)
+				next_word_flag = 1
+				if step_i == 0 : 
+					parent_id =  'bos--1'
+				## if the dict already exists remove it, if it is not a root... 
+				## root?? then next is ''
+				else:
+					next_word_flag = len(gen_sequences_texts[i_bs][step_i]) > step_i ## Not in step_i = 0;
+				if next_word_flag:
+					if not (new_id in id_dict_pos):
+						dict_parent.append({'id': new_id, 'parentId': parent_id , 'text': step_w, 'name': step_w, 'prob' : score })
+						id_dict_pos[new_id] = len(dict_parent) - 1
+				else: 
+					if not (new_id in id_dict_pos):
+						dict_parent.append({'id': new_id, 'parentId': parent_id , 'text': step_w, 'name': step_w, 'prob' : score  })
+						id_dict_pos[new_id] = len(dict_parent) - 1
+
+
+		step_i += 1
+	return dict_parent
+
+
+## Tokenization 
+def compute_tokenization(inputs, targets, w1, model): 
+	colors = ['tok-first-color', 'tok-second-color', 'tok-third-color', 'tok-fourth-color']
+	len_colors = len(colors);
+	inputs = inputs.input_ids
+	html_tokens = ""
+	i = 0 
+	for sentence in inputs: 
+		html_tokens += "<p>"
+		# print("TOKENS", inputs, targets)
+		# print("input", [dict_tokenizer_tr[model].decode(tok) for tok in sentence])
+		tokens = [dict_tokenizer_tr[model].decode(tok) for tok in sentence]
+		for token in tokens:
+			token = token.replace("<", "<'") # .substring(0, token.length - 2) 
+			html_tokens += "<span class='" + colors[i % len_colors] + "'>" + token + " </span>"
+			i +=1
+	html_tokens += "</p>"
+	i = 0 
+	# for tgt_sentence in targets : 
+	html_tokens_tgt = ""
+	html_tokens_tgt += "<p>"
+	# print("targets", [dict_tokenizer_tr[model].decode(tok) for tok in targets])
+	# print("targets", dict_tokenizer_tr[model].decode(targets))
+	tokens = [dict_tokenizer_tr[model].decode(tok) for tok in targets]
+	for token in tokens:
+		token = token.replace("<", "<'") # .substring(0, token.length - 2) 
+		html_tokens_tgt += "<span class='" + colors[i % len_colors] + "'>" + token + " </span>"
+		i +=1
+	html_tokens_tgt += "</p>"
+	# print("HTML", html_tokens, html_tokens_tgt)
+	return html_tokens, html_tokens_tgt
+
+
+def create_vocab_multiple(embeddings_list, model): 
+	"""_summary_
+
+	Args:
+		embeddings_list (list): embedding array 
+
+	Returns:
+		Dict: vocabulary of tokens' embeddings
+	"""
+	print("START VOCAB CREATION MULTIPLE \n \n ")
+	vocab = {} ## add embedds. 
+	sentence_tokens_text_list = []
+	for embeddings in embeddings_list: 
+		tokens_id = embeddings['tokens'] # [[tokens_id]x n_sentences ]
+		for sent_i, sentence in enumerate(tokens_id):
+			sentence_tokens = []
+			for tok_i, token in enumerate(sentence): 
+				sentence_tokens.append(token)
+				if not (token in vocab):
+					vocab[token] = {
+						'token' : token,
+						'count': 1, 
+						# 'text': embeddings['texts'][sent_i][tok_i],
+						'text': dict_tokenizer_tr[model].decode([token]),
+						# 'text': src_token_lists[sent_i][tok_i], 
+						'embed': embeddings['embeddings'][sent_i][tok_i]}
+				else: 
+					vocab[token]['count'] = vocab[token]['count'] + 1  
+		# print(vocab)
+			sentence_tokens_text_list.append(sentence_tokens)
+	print("END VOCAB CREATION MULTIPLE \n \n ")
+	return vocab, sentence_tokens_text_list
+
+def vocab_words_all_prefix(token_embeddings, model, sufix="@@",prefix = '▁' ):
+	vocab = {} 
+	# inf_model = dict_models_tr[model]
+	sentence_words_text_list = []
+	if prefix : 
+		n_prefix = len(prefix)
+		for input_sentences in token_embeddings: 
+			# n_tokens_in_word 
+			for sent_i, sentence in enumerate(input_sentences['tokens']):
+				words_text_list = []
+				# embedding = input_sentences['embed'][sent_i]
+				word = '' 
+				tokens_ids = []
+				embeddings = []
+				ids_to_tokens = dict_tokenizer_tr[model].convert_ids_to_tokens(sentence)
+				# print("validate same len", len(sentence) == len(ids_to_tokens), len(sentence), len(ids_to_tokens), ids_to_tokens)
+
+				to_save= False
+				for tok_i, token_text in enumerate(ids_to_tokens): 
+					token_id = sentence[tok_i]
+					if token_text[:n_prefix] == prefix : 
+						#first we save the previous word 
+						if to_save: 
+							vocab[word] = {
+									'word' : word,
+									'text': word,
+									'count': 1, 
+									'tokens_ids' : tokens_ids, 
+									'embed': np.mean(np.array(embeddings), 0).tolist()
+								}
+							words_text_list.append(word)
+						#word is starting if prefix
+						tokens_ids = [token_id]
+						embeddings = [input_sentences['embeddings'][sent_i][tok_i]]
+						word = token_text[n_prefix:]
+						## if word 
+						to_save = True 
+						
+					else : 
+						if (token_text in dict_tokenizer_tr[model].special_tokens_map.values()):
+							# print('final or save', token_text, token_id, to_save, word)
+							if to_save: 
+								# vocab[word] = ids
+								vocab[word] = {
+									'word' : word,
+									'text': word,
+									'count': 1, 
+									'tokens_ids' : tokens_ids, 
+									'embed': np.mean(np.array(embeddings), 0).tolist()
+								}
+								words_text_list.append(word)
+							#special token is one token element, no continuation 
+							# vocab[token_text] = [token_id]
+							tokens_ids = [token_id]
+							embeddings = [input_sentences['embeddings'][sent_i][tok_i]]
+							vocab[token_text] = {
+									'word' : token_text,
+									'count': 1, 
+									'text': word,
+									'tokens_ids' : tokens_ids, 
+									'embed': np.mean(np.array(embeddings), 0).tolist()
+								}
+							words_text_list.append(token_text)
+							to_save = False
+						else: 
+							# is a continuation; we do not know if it is final; we don't save here.
+							to_save = True 
+							word += token_text 
+							tokens_ids.append(token_id)
+							embeddings.append(input_sentences['embeddings'][sent_i][tok_i])
+				if to_save: 
+					# print('final save', token_text, token_id, to_save, word)
+					vocab[word] = tokens_ids
+					if not (word in vocab):
+						vocab[word] = {
+							'word' : word,
+							'count': 1, 
+							'text': word,
+							'tokens_ids' : tokens_ids, 
+							'embed': np.mean(np.array(embeddings), 0).tolist()
+							}
+						words_text_list.append(word)
+					else: 
+						vocab[word]['count'] = vocab[word]['count'] + 1 
+				sentence_words_text_list.append(words_text_list)
+
+	return vocab, sentence_words_text_list
+def search_query_vocab(index, vocab_queries,  topk = 10, limited_search = []):
+	""" the embed queries are a vocabulary of words : embds_input_voc
+
+	Args:
+		index (_type_): faiss index
+		embed_queries (_type_): vocab format.
+			{   'token' : token,
+				'count': 1, 
+				'text': src_token_lists[sent_i][tok_i], 
+				'embed': embeddings[0]['embeddings'][sent_i][tok_i] }
+		nb_ids (_type_): hash to find the token_id w.r.t the faiss index id. 
+		topk (int, optional): nb of similar tokens. Defaults to 10.
+
+	Returns:
+		_type_: Distance matrix D, indices matrix I and tokens ids (using nb_ids)
+	"""
+	# nb_qi_ids = [] ##ordered ids list
+	nb_q_embds = [] ##ordered embeddings list
+	metadata = {}
+	qi_pos = 0
+	for key , token_values in vocab_queries.items():
+		# nb_qi_ids.append(token_values['token']) # for x in vocab_tokens]
+		metadata[qi_pos] = {'word': token_values['word'], 'tokens': token_values['tokens_ids'], 'text': token_values['text']}
+		qi_pos += 1
+		nb_q_embds.append(token_values['embed']) # for x in vocab_tokens]
+	
+	xq = np.array(nb_q_embds).astype('float32') #elements to query
+
+	D,I = index.search(xq, topk)
+
+	return D,I, metadata
+
+def search_query_vocab_token(index, vocab_queries,  topk = 10, limited_search = []):
+	""" the embed queries are a vocabulary of words : embds_input_vov
+	Returns:
+		_type_: Distance matrix D, indices matrix I and tokens ids (using nb_ids)
+	"""
+	# nb_qi_ids = [] ##ordered ids list
+	nb_q_embds = [] ##ordered embeddings list
+	metadata = {}
+	qi_pos = 0
+	for key , token_values in vocab_queries.items():
+		# nb_qi_ids.append(token_values['token']) # for x in vocab_tokens]
+		metadata[qi_pos] = {'token': token_values['token'], 'text': token_values['text']}
+		qi_pos += 1
+		nb_q_embds.append(token_values['embed']) # for x in vocab_tokens]
+	
+	xq = np.array(nb_q_embds).astype('float32') #elements to query
+
+	D,I = index.search(xq, topk)
+
+	return D,I, metadata	
+
+
+def build_search(query_embeddings, model,type="input"):
+	metadata_all = dict_reference_faiss[model]
+
+	# ## biuld vocab for index 
+	vocab_queries, sentence_tokens_list = create_vocab_multiple(query_embeddings, model)
+	words_vocab_queries, sentence_words_list = vocab_words_all_prefix(query_embeddings, model, sufix="@@",prefix="▁")
+	
+	index_vor_tokens = metadata_all[type]['tokens'][1]
+	md_tokens = metadata_all[type]['tokens'][2]
+	D, I, meta = search_query_vocab_token(index_vor_tokens, vocab_queries)
+
+	qi_pos = 0 
+	similar_tokens = {}
+	# similar_tokens = []
+	for dist, ind in zip(D,I):
+		try: 
+			# similar_tokens.append({
+			similar_tokens[str(meta[qi_pos]['token'])] = {
+				'token': meta[qi_pos]['token'], 
+				'text': meta[qi_pos]['text'], 
+				# 'text': dict_tokenizer_tr[model].decode(meta[qi_pos]['token'])
+				# 'text': meta[qi_pos]['text'], 
+				"similar_topk": [md_tokens[i_index]['token'] for i_index in ind if (i_index != -1) ], 
+				"distance": [dist[i] for (i, i_index) in enumerate(ind) if (i_index != -1)], 
+				}
+			# )
+		except: 
+			print("\n ERROR ", qi_pos, dist, ind)
+		qi_pos += 1
+
+
+	index_vor_words = metadata_all[type]['words'][1]
+	md_words = metadata_all[type]['words'][2]
+
+	Dw, Iw, metaw = search_query_vocab(index_vor_words, words_vocab_queries)
+	# D, I, meta, vocab_words, sentence_words_list = result_input['words']# [2] # D ; I ; meta
+	qi_pos = 0 
+	# similar_words = []
+	similar_words = {}
+	for dist, ind in zip(Dw,Iw):
+		try: 
+			# similar_words.append({
+			similar_words[str(metaw[qi_pos]['word']) ] = {
+				'word': metaw[qi_pos]['word'], 
+				'text': metaw[qi_pos]['word'], 
+				"similar_topk": [md_words[i_index]['word'] for i_index in ind if (i_index != -1) ], 
+				"distance": [dist[i] for (i, i_index) in enumerate(ind) if (i_index != -1)], 
+				}
+			# )
+		except: 
+			print("\n ERROR ", qi_pos, dist, ind)
+		qi_pos += 1
+
+
+	return {'tokens': {'D': D, 'I': I, 'meta': meta, 'vocab_queries': vocab_queries, 'similar':similar_tokens, 'sentence_key_list': sentence_tokens_list}, 
+			'words': {'D':Dw,'I': Iw, 'meta': metaw, 'vocab_queries':words_vocab_queries, 'sentence_key_list': sentence_words_list, 'similar': similar_words}
+			}
+from sklearn.manifold import TSNE
+def embds_input_projection_vocab(vocab, key="token"): 
+	t0 = time()
+	
+	nb_ids = [] ##ordered ids list
+	nb_embds = [] ##ordered embeddings list
+	nb_text = [] ##ordered embeddings list
+	tnse_error = []
+	for _ , token_values in vocab.items():
+		tnse_error.append([0,0])
+		nb_ids.append(token_values[key]) # for x in vocab_tokens]
+		nb_text.append(token_values['text']) # for x in vocab_tokens]
+		nb_embds.append(token_values['embed']) # for x in vocab_tokens]
+
+	X = np.array(nb_embds).astype('float32') #elements to project 
+	try:
+		tsne = TSNE(random_state=0, n_iter=1000)
+		tsne_results = tsne.fit_transform(X)
+	
+		tsne_results = np.c_[tsne_results, nb_ids, nb_text, range(len(nb_ids))] ## creates a zip array : [[TNSE[X,Y], tokenid, token_text], ...]
+	except: 
+		tsne_results = np.c_[tnse_error, nb_ids, nb_text, range(len(nb_ids))] ## creates a zip array : [[TNSE[X,Y], tokenid, token_text], ...]
+
+	t1 = time()
+	print("t-SNE: %.2g sec" % (t1 - t0))
+	# print(tsne_results)
+	
+	return tsne_results.tolist()
+
+
+
+def filtered_projection(similar_key, vocab, model, type="input", key="word"): 
+	metadata_all = dict_reference_faiss[model]
+	vocab_proj = vocab.copy()
+	## tnse projection Input words
+	source_words_voc_similar = set()
+	# for words_set in similar_key:
+	for key_i in similar_key:
+		words_set = similar_key[key_i]
+		source_words_voc_similar.update(words_set['similar_topk'])
+
+	# print(len(source_words_voc_similar))
+	# source_embeddings_filtered = {key:  metadata_all['input']['words'][0][key] for key in source_words_voc_similar}
+	source_embeddings_filtered = {key_value:  metadata_all[type][key][0][key_value] for key_value in source_words_voc_similar}
+	vocab_proj.update(source_embeddings_filtered)
+	## 	vocab_proj add 
+	try:
+		result_TSNE = embds_input_projection_vocab(vocab_proj, key=key[:-1]) ## singular => without 's'
+		dict_projected_embds_all = {str(embds[2]): [embds[0], embds[1], embds[2], embds[3], embds[4]] for embds in result_TSNE}
+	except: 
+		print('TSNE error', type, key)
+		dict_projected_embds_all = {}
+
+	
+
+	# print(result_TSNE)
+	return dict_projected_embds_all 
+
+def get_bertvis_data(input_text, lg_model):
+	tokenizer_tr = dict_tokenizer_tr[lg_model]
+	model_tr = dict_models_tr[lg_model]
+
+	# input_ids = tokenizer_tr(input_text, return_tensors="pt", padding=True)
+	input_ids = tokenizer_tr(input_text, return_tensors="pt", padding=False)
+	result_att = model_tr.generate(**input_ids,
+		num_beams=4,
+        num_return_sequences=4,
+		return_dict_in_generate=True,
+		output_attentions =True,
+		output_scores=True,
+	)
+
+	# tokenizer_tr.convert_ids_to_tokens(result_att.sequences[0])
+	# tokenizer_tr.convert_ids_to_tokens(input_ids.input_ids[0])
+
+	tgt_text = tokenizer_tr.decode(result_att.sequences[0], skip_special_tokens=True)
+
+
+	outputs = model_tr(input_ids=input_ids.input_ids,
+					decoder_input_ids=result_att.sequences[:1],
+					output_attentions =True,
+					)
+	html_attentions = head_view_mod(
+		encoder_attention = outputs.encoder_attentions,
+		cross_attention = outputs.cross_attentions,
+		decoder_attention = outputs.decoder_attentions,
+		encoder_tokens = tokenizer_tr.convert_ids_to_tokens(input_ids.input_ids[0]),
+		decoder_tokens = tokenizer_tr.convert_ids_to_tokens(result_att.sequences[0]),
+		html_action='gradio'
+		)
+	return html_attentions, tgt_text, result_att, outputs
+
+
+def translation_model(w1, model):
+  #translate and get internal values and visualizations; 
+  # src_text = saliency_examples[0]
+  inputs = dict_tokenizer_tr[model](w1, return_tensors="pt", padding=True)
+
+  num_ret_seq = 4
+  translated  = dict_models_tr[model].generate(**inputs, 
+                  num_beams=4,
+                  num_return_sequences=num_ret_seq,
+                  return_dict_in_generate=True, 
+                  output_attentions =True,  
+                  output_hidden_states = True, 
+                  output_scores=True,)
+
+
+  beam_dict = split_token_from_sequences(translated.sequences,model )
+
+  tgt_text = dict_tokenizer_tr[model].decode(translated.sequences[0], skip_special_tokens=True)
+  ## Attentions
+  outputs = dict_models_tr[model](input_ids=inputs.input_ids,
+    decoder_input_ids=translated.sequences[:1],
+    output_attentions =True,
+  )
+  encoder_tokens = dict_tokenizer_tr[model].convert_ids_to_tokens(inputs.input_ids[0])
+  decoder_tokens = dict_tokenizer_tr[model].convert_ids_to_tokens(translated.sequences[0]) 
+#   decoder_tokens = [tok for tok in decoder_tokens if tok != '<pad>']
+#   decoder_tokens = [tok for tok in decoder_tokens if tok != '<pad>']
+
+  # html_attentions = head_view_mod(
+	# 	encoder_attention = outputs.encoder_attentions,
+	# 	cross_attention = outputs.cross_attentions,
+	# 	decoder_attention = outputs.decoder_attentions,
+	# 	encoder_tokens = encoder_tokens,
+	# 	decoder_tokens = decoder_tokens,
+	# 	html_action='gradio'
+	# 	)
+  
+  html_attentions_enc = head_view_mod(
+		encoder_attention = outputs.encoder_attentions,
+		encoder_tokens = encoder_tokens,
+		decoder_tokens = decoder_tokens,
+		html_action='gradio'
+		)
+  
+  html_attentions_dec = head_view_mod(
+		# encoder_attention = outputs.encoder_attentions,
+    decoder_attention = outputs.decoder_attentions,
+		encoder_tokens = encoder_tokens,
+		decoder_tokens = decoder_tokens,
+		html_action='gradio'
+		)
+  
+  html_attentions_cross = head_view_mod(
+    cross_attention = outputs.cross_attentions,
+    encoder_tokens = encoder_tokens,
+    decoder_tokens = decoder_tokens,
+    html_action='gradio'
+    )
+
+  # tokenization 
+  html_in, html_out = compute_tokenization(inputs, translated.sequences[0],w1, model)
+
+  transition_scores = dict_models_tr[model].compute_transition_scores(
+	translated.sequences, translated.scores, translated.beam_indices  , normalize_logits=True
+	)
+  prob_tokens = get_k_prob_tokens(transition_scores, translated, model, k_values=10)
+
+  input_embeddings = dict_models_tr[model].get_encoder().embed_tokens(inputs.input_ids)
+  target_embeddings = dict_models_tr[model].get_decoder().embed_tokens(translated.sequences)
+
+
+  return [tgt_text,
+          [beam_dict,prob_tokens, html_in, html_out, translated, inputs.input_ids,input_embeddings,target_embeddings],
+          [html_attentions_enc['params'], html_attentions_enc['html2'].data], 
+          [html_attentions_dec['params'], html_attentions_dec['html2'].data], 
+          [html_attentions_cross['params'], html_attentions_cross['html2'].data] ]
+
+
+
+html = """
+<html>
+<script async src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>
+
+  <style>
+  .tok-first-color {
+    background: #e0ffcd;
+  }
+
+  .tok-second-color {
+    background: #fdffcd;
+  }
+
+  .tok-third-color {
+    background: #ffebbb;
+  }
+
+  .tok-fourth-color {
+    background: #ffcab0;
+  }
+</style>    
+  <body>
+  
+    <p id="demo"></p>
+    <p id="viz"></p>
+
+    <p id="demo2"></p>
+    <h4> Exploring top-k probable tokens </h4>
+    <div id="d3_text_grid">... top 10 tokens generated at each step ...</div>
+
+    <h4> Exploring the Beam Search sequence generation</h4>
+    <div id="d3_beam_search">... top 4 generated sequences using Beam Search...</div>
+
+  </body>
+</html>
+"""
+
+html_tok = """
+ <div id="d3_tok">... tokenization visualization ...</div> 
+ """
+
+html_embd = """
+ <div id="d3_embd">... token embeddings visualization ...</div>
+	<div id="select_div">
+    	<select id="select_type" class="form-select" aria-label="select example" hidden>
+      	<option selected value="words">Words</option>
+      	<option value="tokens">Tokens</option>
+    	</select>
+  	</div>
+	  <div class="row">
+		<div class="col-9"> 
+		<div id="d3_graph_input_words" class="d3_graph words"></div>
+		</div>
+		<div class="col-3">
+		<div id="similar_input_words" class=""></div>
+		</div>
+		</div>
+	<div id="d3_graph_input_tokens" class="d3_graph tokens"></div>
+	<div id="similar_input_tokens" class=" "></div>
+
+ """
+
+html_tok_target ="""
+ <div id="d3_tok_target">... tokenization visualization ...</div> 
+ """
+
+html_embd_target= """
+ <div id="d3_embd_target">... token embeddings visualization ...</div> 
+ <div id="d3_graph_output_words" class="d3_graph words"></div>
+	<div id="d3_graph_output_tokens" class="d3_graph tokens"></div>
+  <div id="similar_output_words" class=""></div>
+  <div id="similar_output_tokens" class=" "></div>
+ """
+
+html_att_enc =  """
+ <div id="d3_att_enc">... Encoder self attention only -- last layer and mean across heads ... Always read from left to right</div> 
+ <div id="bertviz_enc"></div>
+ """
+
+html_att_cross =  """
+ <div id="d3_att_cross">... Encoder-decoder cross attention only -- last layer and mean across heads ...</div> 
+ """
+
+html_att_dec =  """
+ <div id="d3_att_dec">... decoder self attention only -- last layer and mean across heads ...</div> 
+ """
+
+
+
+def sentence_maker2(w1,j2):
+   print(w1,j2)
+   return "in sentence22..."
+
+
+def first_function(w1, model):
+	global metadata_all
+	#translate and get internal values
+	sentences = w1.split("\n")
+	all_sentences = []
+	translated_text = ''
+	input_embeddings = []
+	output_embeddings = []
+	for sentence in sentences :
+		# print(sentence, end=";") 
+		params = translation_model(sentence, model)
+		all_sentences.append(params)
+		# print(len(params))
+		translated_text +=  params[0] + ' \n'
+		input_embeddings.append({	
+			'embeddings': params[1][6].detach(), ## create a vocabulary with the set of embeddings 
+			'tokens': params[1][3+2].tolist(), # one translation = one sentence
+			# 'texts' : 	dict_tokenizer_tr[model].decode(params[2].tolist())
+
+		}) 
+		output_embeddings.append({
+			'embeddings' : params[1][7].detach(),
+			'tokens': params[1][3+1].sequences.tolist(),
+			# 'texts' : 	dict_tokenizer_tr[model].decode(params[1].sequences.tolist())
+		})
+	
+	## load_reference; 
+	## Build FAISS index 
+	# ---> preload faiss using the respective model with a initial dataset. 
+	## dict_reference_faiss[model] = metadata_all [per language]
+	# result_input = build_reference(input_embeddings,model)
+	# result_output = build_reference(output_embeddings,model)
+	# metadata_all = {'input': result_input, 'output': result_output}
+
+	## Build FAISS index 
+	# ---> preload faiss using the respective model with a initial dataset. 
+	result_search = {}
+	result_search['input'] = build_search(input_embeddings, model, type='input')
+	result_search['output'] = build_search(output_embeddings, model, type='output')
+
+	json_out = {'input': {'tokens': {}, 'words': {}}, 'output': {'tokens': {}, 'words': {}}}
+	dict_projected = {}
+	for type in ['input', 'output']:
+		dict_projected[type] = {} 
+		for key in ['tokens', 'words']: 
+			similar_key = result_search[type][key]['similar']
+			vocab = result_search[type][key]['vocab_queries']
+			dict_projected[type][key] =  filtered_projection(similar_key, vocab, model, type=type, key=key)
+			json_out[type][key]['similar_queries'] = similar_key
+			json_out[type][key]['tnse'] = dict_projected[type][key]
+			json_out[type][key]['key_text_list'] = result_search[type][key]['sentence_key_list']
+
+	## bertviz
+	# paramsbv, tgtbv = get_bertvis_data(w1, model)
+	
+	# params.append(json_out)
+	html_att_enc = params[2][1]#.root_div_id = "bertviz_enc"
+	html_att_dec = params[3][1]
+	html_att_cross = params[4][1]
+	
+
+	params = [params[0], params[1], json_out, params[2][0], params[3][0], params[4][0]]
+	# params.append([tgt, params['params'], params['html2'].data]
+
+	return [translated_text, params, html_att_enc, html_att_dec, html_att_cross]
+
+def second_function(w1,j2):
+	#  json_value = {'one':1}#  return f"{w1['two']} in sentence22..."
+	# to transfer the data to json.
+	print("second_function -- after the js", w1,j2)
+	return "transition to second js function finished."
+
+
+with gr.Blocks(js="plotsjs.js") as demo:
+	gr.Markdown(
+	"""
+	# MAKE NMT Workshop \t `Literacy task` 
+	""")
+
+	gr.Markdown(
+		"""
+		### Translation 
+		""")
+	
+	gr.Markdown(
+		"""
+		1. Select the language pair for the translation 
+		""")
+	radio_c = gr.Radio(choices=['en-zh', 'en-es', 'en-fr', 'en-sw'], value="en-es", label= '', container=False)
+	gr.Markdown(
+		"""
+		2. Source text to translate
+		""")
+	in_text = gr.Textbox(label="source text")
+	with gr.Accordion("Optional: Challenge selection:", open=False):	
+		gr.Markdown(
+			"""
+			### select an example from the challenge set listed bellow
+			""")
+		challenge_ex  = gr.Textbox(label="Challenge", interactive=False)
+		category_minor  = gr.Textbox(label="category_minor", interactive=False)
+		category_major  = gr.Textbox(label="category_major", interactive=False)
+		
+		with gr.Accordion("Examples:"):
+			gr.Examples(arr_challenge_set,[in_text, challenge_ex,category_minor,category_major], label="")
+
+	btn = gr.Button("Translate")
+	
+	
+	with gr.Accordion("3. Review the source tokenization:", open=False):
+		input_tokenisation = gr.HTML(html_tok)
+
+	with gr.Accordion("4. Review similar source tokens in the embedding space:", open=False):
+		input_embd= gr.HTML(html_embd)
+
+	with gr.Accordion("5. Review the attention between the source tokens:", open=False):
+		gr.Markdown(
+			"""
+			`Bertviz ` 
+			""")
+		input_embd= gr.HTML(html_att_enc)
+		enc_html = gr.HTML()
+
+	gr.Markdown(
+		"""
+		### Text is translated into Target Language 
+		""")
+	out_text  = gr.Textbox(label="target text")
+
+	with gr.Accordion("1. Review the target tokenization:", open=False):
+		target_tokenisation = gr.HTML(html_tok_target)
+
+	with gr.Accordion("2. Review similar target tokens in the embedding space:", open=False):
+		target_embd= gr.HTML(html_embd_target)
+
+	with gr.Accordion("3. Review the attention between the target and source tokens:", open=False):
+		gr.Markdown(
+			"""
+			`Bertviz -cross attention` 
+			""")
+		input_embd= gr.HTML(html_att_cross)
+		cross_html = gr.HTML()
+
+	with gr.Accordion("4. Review the attention between the target tokens:", open=False):
+		gr.Markdown(
+			"""
+			`Bertviz -dec attention` 
+			""")
+		input_embd= gr.HTML(html_att_dec)
+		dec_html = gr.HTML()
+
+	with gr.Accordion("6. Review the alternative translations tokens:", open=False):
+		gr.Markdown(
+			"""
+			Generation process : `topk - beam search ` 
+			""")
+		input_mic = gr.HTML(html)
+
+
+	out_text2  = gr.Textbox(visible=False)
+	var2 = gr.JSON(visible=False)
+		
+
+	btn.click(first_function, [in_text, radio_c], [out_text,var2,enc_html, dec_html, cross_html], js="(in_text,radio_c) => testFn_out(in_text,radio_c)") #should return an output comp.
+	out_text.change(second_function, [out_text, var2], out_text2, js="(out_text,var2) => testFn_out_json(var2)") #
+
+	# run script function on load,
+	# demo.load(None,None,None,js="plotsjs.js")
+
+if __name__ == "__main__":   
+    demo.launch()

bertviz_gradio.py

@@ -0,0 +1,251 @@
+
+import json
+import os
+import uuid
+
+from IPython.core.display import display, HTML, Javascript
+
+from bertviz.util import format_special_chars, format_attention, num_layers
+
+print("UP TO DATE")
+
+def head_view_mod(
+        attention=None,
+        tokens=None,
+        sentence_b_start=None,
+        prettify_tokens=True,
+        layer=None,
+        heads=None,
+        encoder_attention=None,
+        decoder_attention=None,
+        cross_attention=None,
+        encoder_tokens=None,
+        decoder_tokens=None,
+        include_layers=None,
+        html_action='view', 
+        patest ="something"
+):
+    """Render head view
+
+        Args:
+            For self-attention models:
+                attention: list of ``torch.FloatTensor``(one for each layer) of shape
+                    ``(batch_size(must be 1), num_heads, sequence_length, sequence_length)``
+                tokens: list of tokens
+                sentence_b_start: index of first wordpiece in sentence B if input text is sentence pair (optional)
+            For encoder-decoder models:
+                encoder_attention: list of ``torch.FloatTensor``(one for each layer) of shape
+                    ``(batch_size(must be 1), num_heads, encoder_sequence_length, encoder_sequence_length)``
+                decoder_attention: list of ``torch.FloatTensor``(one for each layer) of shape
+                    ``(batch_size(must be 1), num_heads, decoder_sequence_length, decoder_sequence_length)``
+                cross_attention: list of ``torch.FloatTensor``(one for each layer) of shape
+                    ``(batch_size(must be 1), num_heads, decoder_sequence_length, encoder_sequence_length)``
+                encoder_tokens: list of tokens for encoder input
+                decoder_tokens: list of tokens for decoder input
+            For all models:
+                prettify_tokens: indicates whether to remove special characters in wordpieces, e.g. Ġ
+                layer: index (zero-based) of initial selected layer in visualization. Defaults to layer 0.
+                heads: Indices (zero-based) of initial selected heads in visualization. Defaults to all heads.
+                include_layers: Indices (zero-based) of layers to include in visualization. Defaults to all layers.
+                    Note: filtering layers may improve responsiveness of the visualization for long inputs.
+                html_action: Specifies the action to be performed with the generated HTML object
+                    - 'view' (default): Displays the generated HTML representation as a notebook cell output
+                    - 'return' : Returns an HTML object containing the generated view for further processing or custom visualization
+    """
+
+    attn_data = []
+    if attention is not None:
+        if tokens is None:
+            raise ValueError("'tokens' is required")
+        if encoder_attention is not None or decoder_attention is not None or cross_attention is not None \
+                or encoder_tokens is not None or decoder_tokens is not None:
+            raise ValueError("If you specify 'attention' you may not specify any encoder-decoder arguments. This"
+                             " argument is only for self-attention models.")
+        if include_layers is None:
+            include_layers = list(range(num_layers(attention)))
+        attention = format_attention(attention, include_layers)
+        if sentence_b_start is None:
+            attn_data.append(
+                {
+                    'name': None,
+                    'attn': attention.tolist(),
+                    'left_text': tokens,
+                    'right_text': tokens
+                }
+            )
+        else:
+            slice_a = slice(0, sentence_b_start)  # Positions corresponding to sentence A in input
+            slice_b = slice(sentence_b_start, len(tokens))  # Position corresponding to sentence B in input
+            attn_data.append(
+                {
+                    'name': 'All',
+                    'attn': attention.tolist(),
+                    'left_text': tokens,
+                    'right_text': tokens
+                }
+            )
+            attn_data.append(
+                {
+                    'name': 'Sentence A -> Sentence A',
+                    'attn': attention[:, :, slice_a, slice_a].tolist(),
+                    'left_text': tokens[slice_a],
+                    'right_text': tokens[slice_a]
+                }
+            )
+            attn_data.append(
+                {
+                    'name': 'Sentence B -> Sentence B',
+                    'attn': attention[:, :, slice_b, slice_b].tolist(),
+                    'left_text': tokens[slice_b],
+                    'right_text': tokens[slice_b]
+                }
+            )
+            attn_data.append(
+                {
+                    'name': 'Sentence A -> Sentence B',
+                    'attn': attention[:, :, slice_a, slice_b].tolist(),
+                    'left_text': tokens[slice_a],
+                    'right_text': tokens[slice_b]
+                }
+            )
+            attn_data.append(
+                {
+                    'name': 'Sentence B -> Sentence A',
+                    'attn': attention[:, :, slice_b, slice_a].tolist(),
+                    'left_text': tokens[slice_b],
+                    'right_text': tokens[slice_a]
+                }
+            )
+    elif encoder_attention is not None or decoder_attention is not None or cross_attention is not None:
+        if encoder_attention is not None:
+            if encoder_tokens is None:
+                raise ValueError("'encoder_tokens' required if 'encoder_attention' is not None")
+            if include_layers is None:
+                include_layers = list(range(num_layers(encoder_attention)))
+            encoder_attention = format_attention(encoder_attention, include_layers)
+            attn_data.append(
+                {
+                    'name': 'Encoder',
+                    'attn': encoder_attention.tolist(),
+                    'left_text': encoder_tokens,
+                    'right_text': encoder_tokens
+                }
+            )
+        if decoder_attention is not None:
+            if decoder_tokens is None:
+                raise ValueError("'decoder_tokens' required if 'decoder_attention' is not None")
+            if include_layers is None:
+                include_layers = list(range(num_layers(decoder_attention)))
+            decoder_attention = format_attention(decoder_attention, include_layers)
+            attn_data.append(
+                {
+                    'name': 'Decoder',
+                    'attn': decoder_attention.tolist(),
+                    'left_text': decoder_tokens,
+                    'right_text': decoder_tokens
+                }
+            )
+        if cross_attention is not None:
+            if encoder_tokens is None:
+                raise ValueError("'encoder_tokens' required if 'cross_attention' is not None")
+            if decoder_tokens is None:
+                raise ValueError("'decoder_tokens' required if 'cross_attention' is not None")
+            if include_layers is None:
+                include_layers = list(range(num_layers(cross_attention)))
+            cross_attention = format_attention(cross_attention, include_layers)
+            attn_data.append(
+                {
+                    'name': 'Cross',
+                    'attn': cross_attention.tolist(),
+                    'left_text': decoder_tokens,
+                    'right_text': encoder_tokens
+                }
+            )
+    else:
+        raise ValueError("You must specify at least one attention argument.")
+
+    if layer is not None and layer not in include_layers:
+        raise ValueError(f"Layer {layer} is not in include_layers: {include_layers}")
+
+    # Generate unique div id to enable multiple visualizations in one notebook
+    vis_id = 'bertviz-%s'%(uuid.uuid4().hex)
+    # vis_id = 'bertviz'#-%s'%(uuid.uuid4().hex)
+
+    # Compose html
+    if len(attn_data) > 1:
+        options = '\n'.join(
+            f'<option value="{i}">{attn_data[i]["name"]}</option>'
+            for i, d in enumerate(attn_data)
+        )
+        select_html = f'Attention: <select id="filter">{options}</select>'
+    else:
+        select_html = ""
+    vis_html = f"""      
+        <div id="{vis_id}" style="font-family:'Helvetica Neue', Helvetica, Arial, sans-serif;">
+            <span style="user-select:none">
+                Layer: <select id="layer"></select>
+                {select_html}
+            </span>
+            <div id='vis'></div>
+        </div>
+    """
+
+    for d in attn_data:
+        attn_seq_len_left = len(d['attn'][0][0])
+        if attn_seq_len_left != len(d['left_text']):
+            raise ValueError(
+                f"Attention has {attn_seq_len_left} positions, while number of tokens is {len(d['left_text'])} "
+                f"for tokens: {' '.join(d['left_text'])}"
+            )
+        attn_seq_len_right = len(d['attn'][0][0][0])
+        if attn_seq_len_right != len(d['right_text']):
+            raise ValueError(
+                f"Attention has {attn_seq_len_right} positions, while number of tokens is {len(d['right_text'])} "
+                f"for tokens: {' '.join(d['right_text'])}"
+            )
+        if prettify_tokens:
+            d['left_text'] = format_special_chars(d['left_text'])
+            d['right_text'] = format_special_chars(d['right_text'])
+    params = {
+        'attention': attn_data,
+        'default_filter': "0",
+        'root_div_id': vis_id,
+        'layer': layer,
+        'heads': heads,
+        'include_layers': include_layers, 
+        'test': 'test'
+    }
+
+    # require.js must be imported for Colab or JupyterLab:
+
+    if html_action == 'gradio': 
+        html1 = HTML('<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>')
+        html2 = HTML(vis_html)
+
+        return {'html1': html1, 'html2' : html2, 'params': params }
+
+
+    if html_action == 'view':
+        display(HTML('<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>'))
+        display(HTML(vis_html))
+        __location__ = os.path.realpath(
+            os.path.join(os.getcwd(), os.path.dirname(__file__)))
+        vis_js = open(os.path.join(__location__, 'head_view.js')).read().replace("PYTHON_PARAMS", json.dumps(params))
+        display(Javascript(vis_js))
+
+    elif html_action == 'return':
+        html1 = HTML('<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.6/require.min.js"></script>')
+
+        html2 = HTML(vis_html)
+
+        __location__ = os.path.realpath(
+            os.path.join(os.getcwd(), os.path.dirname(__file__)))
+        vis_js = open(os.path.join(__location__, 'head_view.js')).read().replace("PYTHON_PARAMS", json.dumps(params))
+        html3 = Javascript(vis_js)
+        script = '\n<script type="text/javascript">\n' + html3.data + '\n</script>\n'
+
+        head_html = HTML(html1.data + html2.data + script)
+        return head_html
+
+    else:
+        raise ValueError("'html_action' parameter must be 'view' or 'return")

challenge_sets.csv

@@ -0,0 +1,109 @@
+name,Lang.,Source sentence,Challenge,category_minor,category_major,Interesting?
+Isabel challenge set,EN,The repeated calls from his mother [should] have alerted us.,Is subject-verb agrement correct? (Possible interference from distractors between the subject's head and the verb).,"S-V agreement, across distractors",Morpho-Syntactic,
+Isabel challenge set,EN,The sudden noise in the upper rooms [should] have alerted us.,Is subject-verb agrement correct? (Possible interference from distractors between the subject's head and the verb).,"S-V agreement, across distractors",Morpho-Syntactic,
+Isabel challenge set,EN,Their repeated failures to report the problem [should] have alerted us.,Is subject-verb agrement correct? (Possible interference from distractors between the subject's head and the verb).,"S-V agreement, across distractors",Morpho-Syntactic,
+Isabel challenge set,EN,She asked her brother not to be [arrogant].,Does the flagged adjective agree correctly with its subject? (Subject-control versus object-control verbs).,"S-V agreement, through control verbs",Morpho-Syntactic,
+Isabel challenge set,EN,She promised her brother not to be [arrogant].,Does the flagged adjective agree correctly with its subject? (Subject-control versus object-control verbs).,"S-V agreement, through control verbs",Morpho-Syntactic,
+Isabel challenge set,EN,She promised her doctor to remain [active] after retiring.,Does the flagged adjective agree correctly with its subject? (Subject-control versus object-control verbs).,"S-V agreement, through control verbs",Morpho-Syntactic,
+Isabel challenge set,EN,My mother promised my father to be more [prudent] on the road.,Does the flagged adjective agree correctly with its subject? (Subject-control versus object-control verbs).,"S-V agreement, through control verbs",Morpho-Syntactic,
+Isabel challenge set,EN,The woman was very [tall] and extremely [strong].,Do the marked verbs/adjective agree correctly with their subject? (Agreement distribution over coordinated predicates),"S-V agreement, coordinated targets",Morpho-Syntactic,
+Isabel challenge set,EN,Their politicians were more [ignorant] than [stupid].,Do the marked verbs/adjective agree correctly with their subject? (Agreement distribution over coordinated predicates),"S-V agreement, coordinated targets",Morpho-Syntactic,
+Isabel challenge set,EN,We [shouted] an insult and [left] abruptly.,Do the marked verbs/adjective agree correctly with their subject? (Agreement distribution over coordinated predicates),"S-V agreement, coordinated targets",Morpho-Syntactic,
+Isabel challenge set,EN,The cat and the dog [should] be [watched].,Do the marked verbs/adjective agree correctly with their subject? (Masculine singular ET masculine singular yields masculine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My father and my brother [will] be [happy] tomorrow.,Do the marked verbs/adjective agree correctly with their subject? (Masculine singular ET masculine singular yields masculine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My book and my pencil [could] be [stolen].,Do the marked verbs/adjective agree correctly with their subject? (Masculine singular ET masculine singular yields masculine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,The cow and the hen [must] be [fed].,Do the marked verbs/adjectives agree correctly with their subject? (Feminine singular ET feminine singular yields feminine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My mother and my sister [will be happy] tomorrow.,Do the marked verbs/adjectives agree correctly with their subject? (Feminine singular ET feminine singular yields feminine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My shoes and my socks [will] be [found].,Do the marked verbs/adjectives agree correctly with their subject? (Feminine singular ET feminine singular yields feminine plural).,"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,The dog and the cow [are] [nervous].,Do the marked verbs/adjectives agree correctly with their subject? (Masculine singular ET feminine singular yields masculine plural.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My father and my mother will be happy tomorrow.,Do the marked verbs/adjectives agree correctly with their subject? (Masculine singular ET feminine singular yields masculine plural.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,My refrigerator and my kitchen table [were] [stolen].,Do the marked verbs/adjectives agree correctly with their subject? (Masculine singular ET feminine singular yields masculine plural.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,Paul and I [could] easily be [convinced] to join you.,Do the marked verbs/adjectives agree correctly with their subject? (Smallest coordinated grammatical person wins.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,You and he [could] be [surprised] by her findings.,Do the marked verbs/adjectives agree correctly with their subject? (Smallest coordinated grammatical person wins.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,We and they [are] on different courses.,Do the marked verbs/adjectives agree correctly with their subject? (Smallest coordinated grammatical person wins.),"S-V agreement, feature calculus on coordinated source",Morpho-Syntactic,
+Isabel challenge set,EN,The woman who [saw] a mouse in the corridor is charming.,Are the agreement marks of the flagged participles the correct ones? (Past participle placed after auxiliary AVOIR agrees with verb object iff object precedes auxiliary. Otherwise participle is in masculine singular form).,"S-V agreement, past participles",Morpho-Syntactic,
+Isabel challenge set,EN,The woman that your brother [saw] in the corridor is charming.,Are the agreement marks of the flagged participles the correct ones? (Past participle placed after auxiliary AVOIR agrees with verb object iff object precedes auxiliary. Otherwise participle is in masculine singular form).,"S-V agreement, past participles",Morpho-Syntactic,
+Isabel challenge set,EN,The house that John has [visited] is crumbling.,Are the agreement marks of the flagged participles the correct ones? (Past participle placed after auxiliary AVOIR agrees with verb object iff object precedes auxiliary. Otherwise participle is in masculine singular form).,"S-V agreement, past participles",Morpho-Syntactic,
+Isabel challenge set,EN,John sold the car that he had [won] in a lottery.,Are the agreement marks of the flagged participles the correct ones? (Past participle placed after auxiliary AVOIR agrees with verb object iff object precedes auxiliary. Otherwise participle is in masculine singular form).,"S-V agreement, past participles",Morpho-Syntactic,
+Isabel challenge set,EN,He will come provided that you [come] too.,"Is the flagged verb in the correct mood? (Certain triggering verbs, adjectives or subordinate conjunctions, induce the subjunctive mood in the subordinate clause that they govern).",Subjunctive mood,Morpho-Syntactic,
+Isabel challenge set,EN,It is unfortunate that he is not [coming] either.,"Is the flagged verb in the correct mood? (Certain triggering verbs, adjectives or subordinate conjunctions, induce the subjunctive mood in the subordinate clause that they govern).",Subjunctive mood,Morpho-Syntactic,
+Isabel challenge set,EN,I requested that families not [be] separated.,"Is the flagged verb in the correct mood? (Certain triggering verbs, adjectives or subordinate conjunctions, induce the subjunctive mood in the subordinate clause that they govern).",Subjunctive mood,Morpho-Syntactic,
+Isabel challenge set,EN,[Mary] sorely misses [Jim].,Are the experiencer and the object of the ``missing'' situation correctly preserved in the French translation? (Argument switch).,Argument switch,Lexico-Syntactic,
+Isabel challenge set,EN,[My sister] is really missing [New York.],Are the experiencer and the object of the ``missing'' situation correctly preserved in the French translation? (Argument switch).,Argument switch,Lexico-Syntactic,
+Isabel challenge set,EN,What [he] misses most is [his dog].,Are the experiencer and the object of the ``missing'' situation correctly preserved in the French translation? (Argument switch).,Argument switch,Lexico-Syntactic,
+Isabel challenge set,EN,John gave [his wonderful wife] a nice present.,Are ``gift'' and ``recipient'' arguments correctly rendered in French? (English double-object constructions),Double-object verbs,Lexico-Syntactic,
+Isabel challenge set,EN,John told [the kids] a nice story.,Are ``gift'' and ``recipient'' arguments correctly rendered in French? (English double-object constructions),Double-object verbs,Lexico-Syntactic,
+Isabel challenge set,EN,John sent [his mother] a nice postcard.,Are ``gift'' and ``recipient'' arguments correctly rendered in French? (English double-object constructions),Double-object verbs,Lexico-Syntactic,
+Isabel challenge set,EN,John [failed to] see the relevance of this point.,Is the meaning of ``fail to'' correctly rendered in the French translation?,Fail to,Lexico-Syntactic,
+Isabel challenge set,EN,He failed to respond.,Is the meaning of ``fail to'' correctly rendered in the French translation?,Fail to,Lexico-Syntactic,
+Isabel challenge set,EN,Those who fail to comply with this requirement will be penalized.,Is the meaning of ``fail to'' correctly rendered in the French translation?,Fail to,Lexico-Syntactic,
+Isabel challenge set,EN,John would like to [swim across] the river.,Is the movement action expressed in the English source correctly rendered in French? (Manner-of-movement verbs with path argument may need to be rephrased in French).,Manner-of-movement verbs,Lexico-Syntactic,
+Isabel challenge set,EN,They [ran into] the room.,Is the movement action expressed in the English source correctly rendered in French? (Manner-of-movement verbs with path argument may need to be rephrased in French).,Manner-of-movement verbs,Lexico-Syntactic,
+Isabel challenge set,EN,The man [ran out of] the park.,Is the movement action expressed in the English source correctly rendered in French? (Manner-of-movement verbs with path argument may need to be rephrased in French).,Manner-of-movement verbs,Lexico-Syntactic,
+Isabel challenge set,EN,John [guitared his way] to San Francisco.,Hard example featuring spontaneous noun-to-verb derivation (``nonce verb'').,Manner-of-movement verbs,Lexico-Syntactic,
+Isabel challenge set,EN,Paul [knows] that this is a fact.,Is the French verb for ``know'' correctly chosen? (Choice between ``savoir''/``connaître'' depends on syntactic nature of its object),Overlapping subcat frames,Lexico-Syntactic,
+Isabel challenge set,EN,Paul [knows] this story.,Is the French verb for ``know'' correctly chosen? (Choice between ``savoir''/``connaître'' depends on syntactic nature of its object),Overlapping subcat frames,Lexico-Syntactic,
+Isabel challenge set,EN,Paul [knows] this story is hard to believe.,Is the French verb for ``know'' correctly chosen? (Choice between ``savoir''/``connaître'' depends on syntactic nature of its object),Overlapping subcat frames,Lexico-Syntactic,
+Isabel challenge set,EN,He [knows] my sister will not take it.,Is the French verb for ``know'' correctly chosen? (Choice between ``savoir''/``connaître'' depends on syntactic nature of its object),Overlapping subcat frames,Lexico-Syntactic,
+Isabel challenge set,EN,My sister [knows] your son is reliable.,Is the French verb for ``know'' correctly chosen? (Choice between ``savoir''/``connaître'' depends on syntactic nature of its object),Overlapping subcat frames,Lexico-Syntactic,
+Isabel challenge set,EN,John believes [Bill to be dishonest].,Is the English ``NP to VP'' complement correctly rendred in the French translation? (Sometimes one needs to translate this structure as a finite clause).,NP to VP,Lexico-Syntactic,
+Isabel challenge set,EN,He liked [his father to tell him stories].,Is the English ``NP to VP'' complement correctly rendred in the French translation? (Sometimes one needs to translate this structure as a finite clause).,NP to VP,Lexico-Syntactic,
+Isabel challenge set,EN,She wanted [her mother to let her go].,Is the English ``NP to VP'' complement correctly rendred in the French translation? (Sometimes one needs to translate this structure as a finite clause).,NP to VP,Lexico-Syntactic,
+Isabel challenge set,EN,John [cooked] a big chicken.,Is the English verb correctly rendered in the French translation? (Agentive use of some French verbs require embedding under ``faire'').,Factitives,Lexico-Syntactic,
+Isabel challenge set,EN,John [melted] a lot of ice.,Is the English verb correctly rendered in the French translation? (Agentive use of some French verbs require embedding under ``faire'').,Factitives,Lexico-Syntactic,
+Isabel challenge set,EN,She likes to [grow] flowers.,Is the English verb correctly rendered in the French translation? (Agentive use of some French verbs require embedding under ``faire'').,Factitives,Lexico-Syntactic,
+Isabel challenge set,EN,Use the meat knife.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Use the butter knife.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Use the steak knife.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the water filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the juice filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the tea filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the cloth filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the metal filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Clean the paper filter.,Is the English nominal compound rendered with the right preposition in the French translation?,Noun Compounds,Lexico-Syntactic,
+Isabel challenge set,EN,Stop [beating around the bush].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,You are [putting the cart before the horse].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,His comment proved to be [the straw that broke the camel's back].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,His argument really [hit the nail on the head].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,It's [no use crying over spilt milk].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,It is [no use crying over spilt milk].,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Common idioms,Lexico-Syntactic,
+Isabel challenge set,EN,The cart has been put before the horse.,Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Syntactically flexible idioms,Lexico-Syntactic,
+Isabel challenge set,EN,"With this argument, [the nail has been hit on the head].",Is the English idiomatic expression correctly rendered with a suitable French idiomatic expression?,Syntactically flexible idioms,Lexico-Syntactic,
+Isabel challenge set,EN,[Have the kids] ever watched that movie?,Is the English question correctly rendered as a French question?,Yes-no question syntax,Syntactic,
+Isabel challenge set,EN,[Hasn't your boss denied you] a promotion?,Is the English question correctly rendered as a French question?,Yes-no question syntax,Syntactic,
+Isabel challenge set,EN,[Shouldn't I attend] this meeting?,Is the English question correctly rendered as a French question?,Yes-no question syntax,Syntactic,
+Isabel challenge set,EN,"Mary looked really happy tonight, [didn't she]?",Is the English ``tag question'' element correctly rendered in the translation?,Tag questions,Syntactic,
+Isabel challenge set,EN,"We should not do that again, [should we]?",Is the English ``tag question'' element correctly rendered in the translation?,Tag questions,Syntactic,
+Isabel challenge set,EN,"She was perfect tonight, [was she not]?",Is the English ``tag question'' element correctly rendered in the translation?,Tag questions,Syntactic,
+Isabel challenge set,EN,The guy [that] she is going out [with] is handsome.,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,[Whom] is she going out [with] these days?,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,The girl [that] he has been talking [about] is smart.,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,[Who] was he talking [to] when you left?,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,The city [that] he is arriving [from] is dangerous.,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,[Where] is he arriving [from]?,Is the dangling preposition of the English sentence correctly placed in the French translation?,WH-MVT and stranded preps,Syntactic,
+Isabel challenge set,EN,Rarely [did the dog] run.,Is the adverb-triggered subject-verb inversion in the English sentence correctly rendered in the French translation?,Adverb-triggered inversion,Syntactic,
+Isabel challenge set,EN,Never before [had she been] so unhappy.,Is the adverb-triggered subject-verb inversion in the English sentence correctly rendered in the French translation?,Adverb-triggered inversion,Syntactic,
+Isabel challenge set,EN,Nowhere [were the birds] so colorful.,Is the adverb-triggered subject-verb inversion in the English sentence correctly rendered in the French translation?,Adverb-triggered inversion,Syntactic,
+Isabel challenge set,EN,Soup [is eaten] with a large spoon.,Is the generic statement made in the English sentence correctly and naturally rendered in the French translation?,Middle voice,Syntactic,
+Isabel challenge set,EN,Masonry [is cut] using a diamond blade.,Is the generic statement made in the English sentence correctly and naturally rendered in the French translation?,Middle voice,Syntactic,
+Isabel challenge set,EN,Champagne [is drunk] in a glass called a flûte.,Is the generic statement made in the English sentence correctly and naturally rendered in the French translation?,Middle voice,Syntactic,
+Isabel challenge set,EN,"[Should] Paul leave, I would be sad.",Fronted ``should'' is interpreted as a conditional subordinator. It is normally translated as ``si'' with imperfect tense.,Fronted ``should'',Syntactic,
+Isabel challenge set,EN,"Should he become president, she would be promoted immediately.",Fronted ``should'' is interpreted as a conditional subordinator. It is normally translated as ``si'' with imperfect tense.,Fronted ``should'',Syntactic,
+Isabel challenge set,EN,"[Should] he fall, he would get up again immediately.",Fronted ``should'' is interpreted as a conditional subordinator. It is normally translated as ``si'' with imperfect tense.,Fronted ``should'',Syntactic,
+Isabel challenge set,EN,She had a lot of money but he did not have [any].,Are the English pronouns correctly rendered in the French translations?,Clitic pronouns,Syntactic,
+Isabel challenge set,EN,He did not talk [to them] very often.,Are the English pronouns correctly rendered in the French translations?,Clitic pronouns,Syntactic,
+Isabel challenge set,EN,The men are watching [each other].,Are the English pronouns correctly rendered in the French translations?,Clitic pronouns,Syntactic,
+Isabel challenge set,EN,He gave [it] to the man.,Are the English pronouns correctly rendered in the French translations?,Clitic pronouns,Syntactic,
+Isabel challenge set,EN,He did not give [it] to [her].,Are the English pronouns correctly rendered in the French translations?,Clitic pronouns,Syntactic,
+Isabel challenge set,EN,The [first four] men were exhausted.,Is the relative order of the ordinals and numerals correct in the French tranlation?,Ordinal placement,Syntactic,
+Isabel challenge set,EN,The [last three] candidates were eliminated.,Is the relative order of the ordinals and numerals correct in the French tranlation?,Ordinal placement,Syntactic,
+Isabel challenge set,EN,The [other two] guys left without paying.,Is the relative order of the ordinals and numerals correct in the French tranlation?,Ordinal placement,Syntactic,
+Isabel challenge set,EN,He washed [his] hands.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,I brushed [my] teeth.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,You brushed [your] teeth.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,I raised [my] hand.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,He turned [his] head.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,He raised his eyes to heaven.,Is the French translation correct and natural both in: a) its use of a particular determiner on the body part noun; and b) the presence or absence of a reflexive pronoun before the verb?,Inalienable possession,Syntactic,
+Isabel challenge set,EN,The strangers [] the woman saw were working.,Is the English zero relative pronoun correctly translated as a non-zero one in the French translation?,Zero REL PRO,Syntactic,
+Isabel challenge set,EN,The man [] your sister hates is evil.,Is the English zero relative pronoun correctly translated as a non-zero one in the French translation?,Zero REL PRO,Syntactic,
+Isabel challenge set,EN,The girl [] my friend was talking about is gone.,Is the English zero relative pronoun correctly translated as a non-zero one in the French translation?,Zero REL PRO,Syntactic,

index/en-es_metadata_ref.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bc8b47dc5db98c9cee79be4647d6853756838632ea2f42c1a4c377e948fd8a3
+size 7599198

plotsjs.js

@@ -0,0 +1,990 @@
+
+
+async () => {
+   // set testFn() function on globalThis, so you html onlclick can access it
+
+   
+    globalThis.testFn = () => {
+      document.getElementById('demo').innerHTML = "Hello?"
+    }; 
+
+    const d37 = await import("https://cdn.jsdelivr.net/npm/d3@7/+esm");
+    const d3 = await import("https://cdn.jsdelivr.net/npm/d3@5/+esm");
+	const $ = await import("https://cdn.jsdelivr.net/npm/jquery@3.7.1/dist/jquery.min.js");
+	globalThis.$ = $;
+
+    globalThis.d3 = d3;
+    
+    globalThis.d3Fn = () => {
+		d3.select('#viz').append('svg')
+				.append('rect')
+				.attr('width', 50)
+				.attr('height', 50)
+				.attr('fill', 'black')
+				.on('mouseover', function(){d3.select(this).attr('fill', 'red')})
+				.on('mouseout', function(){d3.select(this).attr('fill', 'black')});
+
+    }; 
+
+	globalThis.testFn_out = (val,radio_c) => {
+		// document.getElementById('demo').innerHTML = val
+		console.log(val); 
+		// globalThis.d3Fn(); 
+		return([val,radio_c]);
+	  }; 
+
+	  
+	  globalThis.testFn_out_json = (data) => {
+		console.log(data);
+		var $ = jQuery;
+
+		data_beam = data[1][0]; 
+		data_probs = data[1][1]; 
+		data_html_inputs = data[1][2]; 
+		data_html_target = data[1][3]; 
+		data_embds = data[2];
+
+		attViz(data[3]);
+		attViz(data[4]);
+		attViz(data[5]);
+
+
+		console.log(data_beam, )
+		const idMapping = data_beam.reduce((acc, el, i) => {
+		acc[el.id] = i;
+		return acc;
+		}, {});
+
+		let root;
+		data_beam.forEach(el => {
+		// Handle the root element
+		if (el.parentId === null) {
+			root = el;
+			return;
+		}
+		// Use our mapping to locate the parent element in our data_beam array
+		const parentEl = data_beam[idMapping[el.parentId]];
+		// Add our current el to its parent's `children` array
+		parentEl.children = [...(parentEl.children || []), el];
+		});
+
+
+		// console.log(Tree(root)); 
+		// document.getElementById('d3_beam_search').innerHTML = Tree(root)
+		d3.select('#d3_beam_search').html("");
+		d3.select('#d3_beam_search').append(function(){return  Tree(root);});
+
+		//probabilities; 
+		// 
+		d3.select('#d3_text_grid').html("");
+		d3.select('#d3_text_grid').append(function(){return  TextGrid(data_probs);});
+		// $('#d3_text_grid').html(TextGrid(data)) ; 
+
+		//tokenization; 
+		d3.select('#d3_tok').html(data_html_inputs);
+		d3.select('#d3_tok_target').html(data_html_target);
+
+		//embeddings 
+		d3.select("#d3_embeds_source").html("here");
+		// words or token visualization ? 
+		console.log(d3.select("#select_type").node().value);   
+		d3.select("#select_type").attr("hidden", null); 
+		d3.select("#select_type").on("change", change); 
+		change();
+		// tokens 
+		// network plots; 
+		['input', 'output'].forEach(text_type => {
+			['tokens', 'words'].forEach(text_key => {
+				// console.log(type, key, data[0][text_type]);
+				data_i = data_embds[text_type][text_key]; 
+				embeddings_network([], data_i['tnse'], data_i['similar_queries'], type=text_type +"_"+text_key, )
+			}); 
+		});
+
+		// $('#d3_beam_search').html(Tree(root)) ; 
+
+		return(['string', {}])
+
+	}
+
+	function change() {
+		show_type = d3.select("#select_type").node().value;
+		// hide all 
+		d3.selectAll(".d3_embed").attr("hidden",''); 
+		d3.selectAll(".d3_graph").attr("hidden", '');
+		// show current type;
+		d3.select("#d3_embeds_input_" + show_type).attr("hidden", null); 
+		d3.select("#d3_embeds_output_" + show_type).attr("hidden", null); 
+		d3.select("#d3_graph_input_" + show_type).attr("hidden", null); 
+		d3.select("#d3_graph_output_" + show_type).attr("hidden", null); 
+	}
+
+    function embeddings_network(tokens_text, dict_projected_embds, similar_vocab_queries, type="source", ){
+		// tokens_text : not used; 
+		// dict_projected_embds = tnse 
+        console.log("Each token is a node; distance if in similar list", type ); 
+		console.log(tokens_text, dict_projected_embds, similar_vocab_queries); 
+        // similar_vocab_queries_target[key]['similar_topk']
+
+        var nodes_tokens = {}
+        var nodeHash = {};
+        var nodes = []; // [{id: , label: }]
+        var edges = []; // [{source: , target: weight: }]
+        var edges_ids = []; // [{source: , target: weight: }]
+
+        // similar_vocab_queries {key: {similar_topk : [], distance : []}}
+        console.log('similar_vocab_queries', similar_vocab_queries); 
+		prev_node = ''; 
+        for ([sent_token, value] of Object.entries(similar_vocab_queries)) {
+			// console.log('dict_projected_embds',sent_token, parseInt(sent_token), value, dict_projected_embds); 
+            // sent_token = parseInt(sent_token); // Object.entries assumes key:string;  
+            token_text = dict_projected_embds[sent_token][3]
+            if (!nodeHash[sent_token]) {
+                nodeHash[sent_token] = {id: sent_token, label: token_text, type: 'sentence', type_i: 0};  
+                nodes.push(nodeHash[sent_token]);
+            }
+            sim_tokens = value['similar_topk']
+            dist_tokens = value['distance']
+
+            for (let index = 0; index < sim_tokens.length; index++) {
+                const sim = sim_tokens[index];
+                const dist = dist_tokens[index];
+				
+                token_text_sim = dict_projected_embds[sim][3]
+                if (!nodeHash[sim]) {
+                    nodeHash[sim] = {id: sim, label: token_text_sim, type:'similar', type_i: 1}; 
+                    nodes.push(nodeHash[sim]);
+                }
+                edges.push({source: nodeHash[sent_token], target: nodeHash[sim], weight: dist});
+                edges_ids.push({source: sent_token, target: sim, weight: dist});
+            }
+			
+			if (prev_node != '' ) {
+				edges.push({source: nodeHash[prev_node], target:nodeHash[sent_token], weight: 1}); 
+				edges_ids.push({source: prev_node, target: sent_token, weight: 1}); 
+			}
+			prev_node = sent_token;
+			
+        }
+        console.log("TYPE", type, edges, nodes, edges_ids, similar_vocab_queries)   
+        // d3.select('#d3_graph_input_tokens').html(networkPlot({nodes: nodes, links:edges}, similar_vocab_queries, div_type=type) );
+		// type +"_"+key
+		d3.select('#d3_graph_'+type).html("");
+		d3.select('#d3_graph_'+type).append(function(){return  networkPlot({nodes: nodes, links:edges}, similar_vocab_queries, dict_projected_embds,div_type=type);});
+
+        // $('#d3_embeds_network_target').html(networkPlot({nodes: nodes, links:edges}));
+        // $('#d3_embeds_network_'+type).html(etworkPlot({nodes: nodes, link:edges}));
+    }
+
+	function networkPlot(data, similar_vocab_queries,dict_proj, div_type="source", {
+		width = 400, // outer width, in pixels
+		height , // outer height, in pixels
+		r = 3, // radius of nodes
+		padding = 1, // horizontal padding for first and last column
+		// text = d => d[2],
+	} = {}){
+		// data_dict = data; 
+		data = data// [div_type];
+		similar_vocab_queries = similar_vocab_queries// [div_type];
+		console.log("data, similar_vocab_queries, div_type"); 
+		console.log(data, similar_vocab_queries, div_type); 
+	
+	  // Create the SVG container.
+	  var margin = {top: 10, right: 10, bottom: 30, left: 50 },
+		width = width //- margin.left - margin.right,
+		height = 400 //- margin.top - margin.bottom;
+	
+		width_box = width + margin.left + margin.right; 
+		height_box = height + margin.top + margin.bottom
+		totalWidth = width*2; 
+	
+	
+		var svg = d37.create("svg")
+		.attr("width", width + margin.left + margin.right)
+		.attr("height", height + margin.top + margin.bottom)
+
+		// Initialize the links
+		var link = svg
+			.selectAll("line")
+			.data(data.links)
+			.enter()
+			.append("line")
+			.style("fill", d => d.weight == 1 ? "#dfd5d5" :  "#000000") // , "#69b3a2" : "#69b3a2")
+			.style("stroke", "#aaa")
+	
+		
+		
+		var text = svg
+			.selectAll("text")
+			.data(data.nodes)
+			.enter()
+			.append("text")
+			  .style("text-anchor", "middle")
+			  .attr("y", 15)
+			.attr("class", d => 'text_token-'+ dict_proj[d.id][4] + div_type)
+			.attr("div-type", div_type)
+			// .attr("class", d => 'text_token-'+ d.index)
+			  .text(function (d) {return d.label} )
+			//   .on('mouseover', function(d) { (d.type_i == 0) ? highlight_mouseover_text : console.log(0)} )
+			//   .on('mouseover', function(d) { (d.type_i == 0) ? highlight_mouseout_text : '' } )
+			//   .on('mouseout', highlight_mouseout_text )
+			// .join('text')
+			// .text(function(d) {
+				// return d.id
+			// })
+	
+		// Initialize the nodes
+		var node = svg
+			.selectAll("circle")
+			.data(data.nodes)
+			.enter()
+			.append("circle")
+			.attr("r", 6)
+			// .attr("class", d => 'node_token-'+ d.id)
+			.attr("class", d => 'node_token-'+ dict_proj[d.id][4] + div_type)
+			.attr("div-type", div_type)
+			.style("fill", d => d.type_i ? "#e85252" :  "#6689c6") // , "#69b3a2" : "#69b3a2")
+			.on('mouseover', highlight_mouseover )
+			// .on('mouseover', function(d) { return (d.type_i == 0) ? highlight_mouseover : console.log(0)} )
+			.on('mouseout',highlight_mouseout )
+			.on('click', change_legend )
+			// .on('click', show_similar_tokens )
+	
+	
+	
+		// Let's list the force we wanna apply on the network
+		var simulation = d37.forceSimulation(data.nodes)                 // Force algorithm is applied to data.nodes
+			.force("link", d37.forceLink()                               // This force provides links between nodes
+			.id(function(d) { return d.id; })                     // This provide  the id of a node
+			.links(data.links)                                    // and this the list of links
+			)
+			.force("charge", d37.forceManyBody(-400))         // This adds repulsion between nodes. Play with the -400 for the repulsion strength
+			.force("center", d37.forceCenter(width / 2, height / 2))     // This force attracts nodes to the center of the svg area
+			// .force("collision", d3.forceCollide())
+			.on("end", ticked);
+	
+		// This function is run at each iteration of the force algorithm, updating the nodes position.
+		function ticked() {
+			link
+				.attr("x1", function(d) { return d.source.x; })
+				.attr("y1", function(d) { return d.source.y; })
+				.attr("x2", function(d) { return d.target.x; })
+				.attr("y2", function(d) { return d.target.y; });
+	
+			node
+				.attr("cx", function (d) { return d.x+3; })
+				.attr("cy", function(d) { return d.y-3; });
+			
+			text
+				.attr("transform", function(d) { return "translate(" + d.x + "," + d.y + ")"; })
+		}
+	
+		function highlight_mouseover(d,i) {
+			console.log("highlight_mouseover", d,i, d37.select(this).attr("div-type")); 
+			if (i.type_i == 0 ){
+					token_id = i.id
+					similar_ids = similar_vocab_queries[token_id]['similar_topk']; 
+					d37.select(this).transition()
+									.duration('50')
+									.style('opacity', '1')
+									.attr("r", 12)
+					type = d37.select(this).attr("div-type")
+					similar_ids.forEach(similar_token => {
+						node_id_name = dict_proj[similar_token][4]
+						d37.selectAll('.node_token-'+ node_id_name + type).attr("r",12 ).style('opacity', '1')//.raise()
+						// d3.selectAll('.text_token-'+ node_id_name).raise()
+					});
+				}
+		}
+		
+	
+		function highlight_mouseout(d,i) {
+			if (i.type_i == 0 ){
+				token_id = i.id
+				console.log("similar_vocab_queries", similar_vocab_queries, "this type:", d37.select(this).attr("div-type")); 
+				similar_ids = similar_vocab_queries[token_id]['similar_topk']; 
+				// clean_sentences(); 
+				d37.select(this).transition()
+					.duration('50')
+					.style('opacity', '.7')
+					.attr("r", 6)
+				type = d37.select(this).attr("div-type")
+				similar_ids.forEach(similar_token => {
+						node_id_name = dict_proj[similar_token][4]
+						d37.selectAll('.node_token-' + node_id_name + type).attr("r",6 ).style('opacity', '.7')
+						d37.selectAll("circle").raise()
+					});
+			}
+		}
+	
+		function change_legend(d,i,j) {
+			console.log(d,i,dict_proj); 
+			if (i['id'] in dict_proj){
+				// show_sentences(dict_proj[i[2]], i[2]);
+	
+				show_similar_tokens(i['id'], '#similar_'+type); 
+	
+				console.log(dict_proj[i['id']]);
+			}
+			else{console.log("no sentence")}; 
+		}
+	
+		function show_similar_tokens(token, div_name_similar='#similar_input_tokens') {
+			d37.select(div_name_similar).html("");
+			console.log("token", token);
+			console.log("similar_vocab_queries[token]", similar_vocab_queries[token]); 
+			token_data = similar_vocab_queries[token]; 
+			console.log(token, token_data); 
+			var decForm = d37.format(".3f");
+	
+			d37.select(div_name_similar)
+				.selectAll().append("p")
+				.data(token_data['similar_topk'])
+				.enter()
+				.append("p").append('text')
+				// .attr('class_data', sent_id)
+				.attr('class_id', d => d)
+				.style("background", d=> {if (d == token) return "yellow"} )
+				// .text( d => d  + " \n "); 
+				.text((d,i) => do_text(d,i) ); 
+			
+			function do_text(d,i){
+				console.log("do_text d,i" ); 
+				console.log(d,i); 
+				console.log("data_dict[d], data_dict");
+				console.log(dict_proj[d], dict_proj);
+				return dict_proj[d][3] + " " + decForm(token_data['distance'][i]) + "   "; 
+			}
+	
+	
+		}
+	
+	
+		return svg.node();
+	
+		}; 
+	
+
+
+// Copyright 2021 Observable, Inc.
+// Released under the ISC license.
+// https://observablehq.com/@d3/tree
+function Tree(data, { // data is either tabular (array of objects) or hierarchy (nested objects)
+	path, // as an alternative to id and parentId, returns an array identifier, imputing internal nodes
+	id = Array.isArray(data) ? d => d.id : null, // if tabular data, given a d in data, returns a unique identifier (string)
+	parentId = Array.isArray(data) ? d => d.parentId : null, // if tabular data, given a node d, returns its parent’s identifier
+	children, // if hierarchical data, given a d in data, returns its children
+	tree = d3.tree, // layout algorithm (typically d3.tree or d3.cluster)
+	sort, // how to sort nodes prior to layout (e.g., (a, b) => d3.descending(a.height, b.height))
+	label =  d => d.name, // given a node d, returns the display name
+	title = d => d.name, // given a node d, returns its hover text
+	link , // given a node d, its link (if any)
+	linkTarget = "_blank", // the target attribute for links (if any)
+	width = 800, // outer width, in pixels
+	height, // outer height, in pixels
+	r = 3, // radius of nodes
+	padding = 1, // horizontal padding for first and last column
+	fill = "#999", // fill for nodes
+	fillOpacity, // fill opacity for nodes
+	stroke = "#555", // stroke for links
+	strokeWidth = 2, // stroke width for links
+	strokeOpacity = 0.4, // stroke opacity for links
+	strokeLinejoin, // stroke line join for links
+	strokeLinecap, // stroke line cap for links
+	halo = "#fff", // color of label halo 
+	haloWidth = 3, // padding around the labels
+	curve = d37.curveBumpX, // curve for the link
+  } = {}) {
+  
+	// If id and parentId options are specified, or the path option, use d3.stratify
+	// to convert tabular data to a hierarchy; otherwise we assume that the data is
+	// specified as an object {children} with nested objects (a.k.a. the “flare.json”
+	// format), and use d3.hierarchy.
+	const root = path != null ? d3.stratify().path(path)(data)
+		: id != null || parentId != null ? d3.stratify().id(id).parentId(parentId)(data)
+		: d3.hierarchy(data, children);
+  
+	// Sort the nodes.
+	if (sort != null) root.sort(sort);
+  
+	// Compute labels and titles.
+	const descendants = root.descendants();
+	const L = label == null ? null : descendants.map(d => label(d.data, d));
+  
+	// Compute the layout.
+	const descWidth = 10; 
+	// console.log('descendants', descendants);
+	const realWidth = descWidth * descendants.length
+	const totalWidth = (realWidth > width) ? realWidth : width;
+
+	const dx = 25;
+	const dy = totalWidth / (root.height + padding);
+	tree().nodeSize([dx, dy])(root);
+  
+	// Center the tree.
+	let x0 = Infinity;
+	let x1 = -x0;
+	root.each(d => {
+	  if (d.x > x1) x1 = d.x;
+	  if (d.x < x0) x0 = d.x;
+	});
+  
+	// Compute the default height.
+	if (height === undefined) height = x1 - x0 + dx * 2;
+
+
+  
+	// Use the required curve
+	if (typeof curve !== "function") throw new Error(`Unsupported curve`);
+  
+	const parent = d3.create("div");
+	
+	const body = parent.append("div")
+	.style("overflow-x", "scroll")
+	.style("-webkit-overflow-scrolling", "touch");
+	
+	const svg = body.append("svg")
+		.attr("viewBox", [-dy * padding / 2, x0 - dx, totalWidth, height])
+		.attr("width", totalWidth)
+		.attr("height", height)
+		.attr("style", "max-width: 100%; height: auto; height: intrinsic;")
+		.attr("font-family", "sans-serif")
+		.attr("font-size", 12);
+  
+	svg.append("g")
+		.attr("fill", "none")
+		.attr("stroke", stroke)
+		.attr("stroke-opacity", strokeOpacity)
+		.attr("stroke-linecap", strokeLinecap)
+		.attr("stroke-linejoin", strokeLinejoin)
+		.attr("stroke-width", strokeWidth)
+	  .selectAll("path")
+		.data(root.links())
+		.join("path")
+		// .attr("stroke", d => d.prob > 0.5 ? 'red' : 'blue'  )
+		// .attr("fill", "red")
+		  .attr("d", d37.link(curve)
+			  .x(d => d.y)
+			  .y(d => d.x));
+  
+	const node = svg.append("g")
+	  .selectAll("a")
+	  .data(root.descendants())
+	  .join("a")
+		.attr("xlink:href", link == null ? null : d => link(d.data, d))
+		.attr("target", link == null ? null : linkTarget)
+		.attr("transform", d => `translate(${d.y},${d.x})`);
+  
+	node.append("circle")
+		.attr("fill", d => d.children ? stroke : fill)
+		.attr("r", r);
+  
+	title = d => (d.name + ( d.prob)); 
+
+	if (title != null) node.append("title")
+		.text(d => title(d.data, d));
+  
+	if (L) node.append("text")
+		.attr("dy", "0.32em")
+		.attr("x", d => d.children ? -6 : 6)
+		.attr("text-anchor", d => d.children ? "end" : "start")
+		.attr("paint-order", "stroke")
+		.attr("stroke", 'white')
+		.attr("fill", d => d.data.prob == 1 ? ('red') : ('black')  )
+		.attr("stroke-width", haloWidth)
+		.text((d, i) => L[i]);
+		body.node().scrollBy(totalWidth, 0);
+	return svg.node();
+  }
+
+  function TextGrid(data, div_name, {
+	width = 640, // outer width, in pixels
+	height , // outer height, in pixels
+	r = 3, // radius of nodes
+	padding = 1, // horizontal padding for first and last column
+	// text = d => d[2],
+} = {}){
+	// console.log("TextGrid", data); 
+
+	// Compute the layout.
+	const dx = 10;
+	const dy = 10;  //width / (root.height + padding);
+	
+	const marginTop = 20;
+	const marginRight = 20;
+	const marginBottom = 30;
+	const marginLeft = 30;
+
+	// Center the tree.
+	let x0 = Infinity;
+	let x1 = -x0;
+	topk = 10; 
+	word_length = 20;
+	const rectWidth = 60; 
+	const rectTotal = 70; 
+
+	wval = 0
+	
+	const realWidth = rectTotal * data.length
+	const totalWidth = (realWidth > width) ? realWidth : width;
+	// root.each(d => {
+	// 	if (d.x > x1) x1 = d.x;
+	// 	if (d.x < x0) x0 = d.x;
+	// });
+	
+	// Compute the default height.
+	// if (height === undefined) height = x1 - x0 + dx * 2;
+	if (height === undefined) height = topk * word_length + 10;
+
+	const parent = d3.create("div");
+
+	// parent.append("svg")
+    //   .attr("width", width)
+    //   .attr("height", height)
+    //   .style("position", "absolute")
+    //   .style("pointer-events", "none")
+    //   .style("z-index", 1);
+
+
+	// const svg = d3.create("svg")
+	// // svg = parent.append("svg")
+	// 	.attr("viewBox", [-dy * padding / 2, x0 - dx, width, height])
+	// 	.attr("width", width)
+	// 	.attr("height", height)
+	// 	.attr("style", "max-width: 100%; height: auto; height: intrinsic;")
+	// 	.attr("font-family", "sans-serif")
+	// 	.attr("font-size", 10);
+
+	// div.data([1, 2, 4, 8, 16, 32], d => d);
+	// div.enter().append("div").text(d => d);
+
+	const body = parent.append("div")
+	.style("overflow-x", "scroll")
+	.style("-webkit-overflow-scrolling", "touch");
+
+	const svg = body.append("svg")
+		.attr("width", totalWidth)
+		.attr("height", height)
+		.style("display", "block")
+		.attr("font-family", "sans-serif")
+		.attr("font-size", 10);
+
+
+		data.forEach(words_list => {
+			// console.log(wval, words_list); 
+			words = words_list[2]; // {'t': words_list[2], 'p': words_list[1]}; 
+			scores =  words_list[1];
+			words_score = words.map( (x,i) => {return {t: x, p: scores[i]}})
+			// console.log(words_score); 
+			// svg.selectAll("text").enter()
+			// 	.data(words)
+			// 	.join("text")
+			// 	.text((d,i) => (d))
+			// 	.attr("x", wval) 
+			// 	.attr("y", ((d,i) => (20 + i*20)))
+
+			var probs = svg.selectAll("text").enter()
+				.data(words_score).join('g');
+	
+
+	
+			  probs.append("rect")
+				// .data(words)
+				.attr("x", wval) 
+				.attr("y", ((d,i) => ( 10+ i*20)))
+				.attr('width', rectWidth)
+				.attr('height', 15)
+				.attr("color", 'gray')
+				.attr("fill", "gray")
+				// .attr("fill-opacity", "0.2")
+				.attr("fill-opacity", (d) => (d.p))
+				.attr("stroke-opacity", 0.8)
+				.append("svg:title")
+				.text(function(d){return d.t+":"+d.p;}); 
+
+
+			probs.append("text")
+				// .data(words)
+				.text((d,i) => (d.t))
+				.attr("x", wval) 
+				.attr("y", ((d,i) => (20 + i*20)))
+				// .attr("fill", 'white')
+				.attr("font-weight", 700); 
+
+			wval = wval + rectTotal;
+			});
+
+
+	body.node().scrollBy(totalWidth, 0);
+	// return svg.node();
+	return parent.node();
+  }
+
+
+  function attViz(PYTHON_PARAMS) {
+	var $ = jQuery;
+const params = PYTHON_PARAMS; // HACK: PYTHON_PARAMS is a template marker that is replaced by actual params.
+const TEXT_SIZE = 15;
+const BOXWIDTH = 110;
+const BOXHEIGHT = 22.5;
+const MATRIX_WIDTH = 115;
+const CHECKBOX_SIZE = 20;
+const TEXT_TOP = 30;
+
+console.log("d3 version in ffuntions", d3.version)
+let headColors;
+try {
+	headColors = d3.scaleOrdinal(d3.schemeCategory10);
+} catch (err) {
+	console.log('Older d3 version')
+	headColors = d3.scale.category10();
+}
+let config = {};
+// globalThis.
+initialize();
+renderVis();
+
+function initialize() {
+	// globalThis.initialize = () => {
+
+		console.log("init")
+	config.attention = params['attention'];
+	config.filter = params['default_filter'];
+	config.rootDivId = params['root_div_id'];
+	config.nLayers = config.attention[config.filter]['attn'].length;
+	config.nHeads = config.attention[config.filter]['attn'][0].length;
+	config.layers = params['include_layers']
+
+	if (params['heads']) {
+		config.headVis = new Array(config.nHeads).fill(false);
+		params['heads'].forEach(x => config.headVis[x] = true);
+	} else {
+		config.headVis = new Array(config.nHeads).fill(true);
+	}
+	config.initialTextLength = config.attention[config.filter].right_text.length;
+	config.layer_seq = (params['layer'] == null ? 0 : config.layers.findIndex(layer => params['layer'] === layer));
+	config.layer = config.layers[config.layer_seq]
+	
+	// '#' + temp1.root_div_id+ ' #layer'
+	$('#' + config.rootDivId+ ' #layer').empty();
+	
+	let layerEl = $('#' + config.rootDivId+ ' #layer');
+	console.log(layerEl)
+	for (const layer of config.layers) {
+		layerEl.append($("<option />").val(layer).text(layer));
+	}
+	layerEl.val(config.layer).change();
+	layerEl.on('change', function (e) {
+		config.layer = +e.currentTarget.value;
+		config.layer_seq = config.layers.findIndex(layer => config.layer === layer);
+		renderVis();
+	});
+
+	$('#'+config.rootDivId+' #filter').on('change', function (e) {
+	// $(`#${config.rootDivId} #filter`).on('change', function (e) {
+
+		config.filter = e.currentTarget.value;
+		renderVis();
+	});
+}
+
+function renderVis() {
+
+	// Load parameters
+	const attnData = config.attention[config.filter];
+	const leftText = attnData.left_text;
+	const rightText = attnData.right_text;
+
+	// Select attention for given layer
+	const layerAttention = attnData.attn[config.layer_seq];
+
+	// Clear vis
+	$('#'+config.rootDivId+' #vis').empty();
+
+	// Determine size of visualization
+	const height = Math.max(leftText.length, rightText.length) * BOXHEIGHT + TEXT_TOP;
+	const svg = d3.select('#'+ config.rootDivId +' #vis')
+		.append('svg')
+		.attr("width", "100%")
+		.attr("height", height + "px");
+
+	// Display tokens on left and right side of visualization
+	renderText(svg, leftText, true, layerAttention, 0);
+	renderText(svg, rightText, false, layerAttention, MATRIX_WIDTH + BOXWIDTH);
+
+	// Render attention arcs
+	renderAttention(svg, layerAttention);
+
+	// Draw squares at top of visualization, one for each head
+	drawCheckboxes(0, svg, layerAttention);
+}
+
+function renderText(svg, text, isLeft, attention, leftPos) {
+
+	const textContainer = svg.append("svg:g")
+		.attr("id", isLeft ? "left" : "right");
+
+	// Add attention highlights superimposed over words
+	textContainer.append("g")
+		.classed("attentionBoxes", true)
+		.selectAll("g")
+		.data(attention)
+		.enter()
+		.append("g")
+		.attr("head-index", (d, i) => i)
+		.selectAll("rect")
+		.data(d => isLeft ? d : transpose(d)) // if right text, transpose attention to get right-to-left weights
+		.enter()
+		.append("rect")
+		.attr("x", function () {
+			var headIndex = +this.parentNode.getAttribute("head-index");
+			return leftPos + boxOffsets(headIndex);
+		})
+		.attr("y", (+1) * BOXHEIGHT)
+		.attr("width", BOXWIDTH / activeHeads())
+		.attr("height", BOXHEIGHT)
+		.attr("fill", function () {
+			return headColors(+this.parentNode.getAttribute("head-index"))
+		})
+		.style("opacity", 0.0);
+
+	const tokenContainer = textContainer.append("g").selectAll("g")
+		.data(text)
+		.enter()
+		.append("g");
+
+	// Add gray background that appears when hovering over text
+	tokenContainer.append("rect")
+		.classed("background", true)
+		.style("opacity", 0.0)
+		.attr("fill", "lightgray")
+		.attr("x", leftPos)
+		.attr("y", (d, i) => TEXT_TOP + i * BOXHEIGHT)
+		.attr("width", BOXWIDTH)
+		.attr("height", BOXHEIGHT);
+
+	// Add token text
+	const textEl = tokenContainer.append("text")
+		.text(d => d)
+		.attr("font-size", TEXT_SIZE + "px")
+		.style("cursor", "default")
+		.style("-webkit-user-select", "none")
+		.attr("x", leftPos)
+		.attr("y", (d, i) => TEXT_TOP + i * BOXHEIGHT);
+
+	if (isLeft) {
+		textEl.style("text-anchor", "end")
+			.attr("dx", BOXWIDTH - 0.5 * TEXT_SIZE)
+			.attr("dy", TEXT_SIZE);
+	} else {
+		textEl.style("text-anchor", "start")
+			.attr("dx", +0.5 * TEXT_SIZE)
+			.attr("dy", TEXT_SIZE);
+	}
+
+	tokenContainer.on("mouseover", function (d, index) {
+
+		// Show gray background for moused-over token
+		textContainer.selectAll(".background")
+			.style("opacity", (d, i) => i === index ? 1.0 : 0.0)
+
+		// Reset visibility attribute for any previously highlighted attention arcs
+		svg.select("#attention")
+			.selectAll("line[visibility='visible']")
+			.attr("visibility", null)
+
+		// Hide group containing attention arcs
+		svg.select("#attention").attr("visibility", "hidden");
+
+		// Set to visible appropriate attention arcs to be highlighted
+		if (isLeft) {
+			svg.select("#attention").selectAll("line[left-token-index='" + index + "']").attr("visibility", "visible");
+		} else {
+			svg.select("#attention").selectAll("line[right-token-index='" + index + "']").attr("visibility", "visible");
+		}
+
+		// Update color boxes superimposed over tokens
+		const id = isLeft ? "right" : "left";
+		const leftPos = isLeft ? MATRIX_WIDTH + BOXWIDTH : 0;
+		svg.select("#" + id)
+			.selectAll(".attentionBoxes")
+			.selectAll("g")
+			.attr("head-index", (d, i) => i)
+			.selectAll("rect")
+			.attr("x", function () {
+				const headIndex = +this.parentNode.getAttribute("head-index");
+				return leftPos + boxOffsets(headIndex);
+			})
+			.attr("y", (d, i) => TEXT_TOP + i * BOXHEIGHT)
+			.attr("width", BOXWIDTH / activeHeads())
+			.attr("height", BOXHEIGHT)
+			.style("opacity", function (d) {
+				const headIndex = +this.parentNode.getAttribute("head-index");
+				if (config.headVis[headIndex])
+					if (d) {
+						return d[index];
+					} else {
+						return 0.0;
+					}
+				else
+					return 0.0;
+			});
+	});
+
+	textContainer.on("mouseleave", function () {
+
+		// Unhighlight selected token
+		d3.select(this).selectAll(".background")
+			.style("opacity", 0.0);
+
+		// Reset visibility attributes for previously selected lines
+		svg.select("#attention")
+			.selectAll("line[visibility='visible']")
+			.attr("visibility", null) ;
+		svg.select("#attention").attr("visibility", "visible");
+
+		// Reset highlights superimposed over tokens
+		svg.selectAll(".attentionBoxes")
+			.selectAll("g")
+			.selectAll("rect")
+			.style("opacity", 0.0);
+	});
+}
+
+function renderAttention(svg, attention) {
+
+	// Remove previous dom elements
+	svg.select("#attention").remove();
+
+	// Add new elements
+	svg.append("g")
+		.attr("id", "attention") // Container for all attention arcs
+		.selectAll(".headAttention")
+		.data(attention)
+		.enter()
+		.append("g")
+		.classed("headAttention", true) // Group attention arcs by head
+		.attr("head-index", (d, i) => i)
+		.selectAll(".tokenAttention")
+		.data(d => d)
+		.enter()
+		.append("g")
+		.classed("tokenAttention", true) // Group attention arcs by left token
+		.attr("left-token-index", (d, i) => i)
+		.selectAll("line")
+		.data(d => d)
+		.enter()
+		.append("line")
+		.attr("x1", BOXWIDTH)
+		.attr("y1", function () {
+			const leftTokenIndex = +this.parentNode.getAttribute("left-token-index")
+			return TEXT_TOP + leftTokenIndex * BOXHEIGHT + (BOXHEIGHT / 2)
+		})
+		.attr("x2", BOXWIDTH + MATRIX_WIDTH)
+		.attr("y2", (d, rightTokenIndex) => TEXT_TOP + rightTokenIndex * BOXHEIGHT + (BOXHEIGHT / 2))
+		.attr("stroke-width", 2)
+		.attr("stroke", function () {
+			const headIndex = +this.parentNode.parentNode.getAttribute("head-index");
+			return headColors(headIndex)
+		})
+		.attr("left-token-index", function () {
+			return +this.parentNode.getAttribute("left-token-index")
+		})
+		.attr("right-token-index", (d, i) => i)
+	;
+	updateAttention(svg)
+}
+
+function updateAttention(svg) {
+	svg.select("#attention")
+		.selectAll("line")
+		.attr("stroke-opacity", function (d) {
+			const headIndex = +this.parentNode.parentNode.getAttribute("head-index");
+			// If head is selected
+			if (config.headVis[headIndex]) {
+				// Set opacity to attention weight divided by number of active heads
+				return d / activeHeads()
+			} else {
+				return 0.0;
+			}
+		})
+}
+
+function boxOffsets(i) {
+	const numHeadsAbove = config.headVis.reduce(
+		function (acc, val, cur) {
+			return val && cur < i ? acc + 1 : acc;
+		}, 0);
+	return numHeadsAbove * (BOXWIDTH / activeHeads());
+}
+
+function activeHeads() {
+	return config.headVis.reduce(function (acc, val) {
+		return val ? acc + 1 : acc;
+	}, 0);
+}
+
+function drawCheckboxes(top, svg) {
+	const checkboxContainer = svg.append("g");
+	const checkbox = checkboxContainer.selectAll("rect")
+		.data(config.headVis)
+		.enter()
+		.append("rect")
+		.attr("fill", (d, i) => headColors(i))
+		.attr("x", (d, i) => i * CHECKBOX_SIZE)
+		.attr("y", top)
+		.attr("width", CHECKBOX_SIZE)
+		.attr("height", CHECKBOX_SIZE);
+
+	function updateCheckboxes() {
+		checkboxContainer.selectAll("rect")
+			.data(config.headVis)
+			.attr("fill", (d, i) => d ? headColors(i): lighten(headColors(i)));
+	}
+
+	updateCheckboxes();
+
+	checkbox.on("click", function (d, i) {
+		if (config.headVis[i] && activeHeads() === 1) return;
+		config.headVis[i] = !config.headVis[i];
+		updateCheckboxes();
+		updateAttention(svg);
+	});
+
+	checkbox.on("dblclick", function (d, i) {
+		// If we double click on the only active head then reset
+		if (config.headVis[i] && activeHeads() === 1) {
+			config.headVis = new Array(config.nHeads).fill(true);
+		} else {
+			config.headVis = new Array(config.nHeads).fill(false);
+			config.headVis[i] = true;
+		}
+		updateCheckboxes();
+		updateAttention(svg);
+	});
+}
+
+function lighten(color) {
+	const c = d3.hsl(color);
+	const increment = (1 - c.l) * 0.6;
+	c.l += increment;
+	c.s -= increment;
+	return c;
+}
+
+function transpose(mat) {
+	return mat[0].map(function (col, i) {
+		return mat.map(function (row) {
+			return row[i];
+		});
+	});
+}
+
+}
+
+
+
+
+
+}
+

requirements.txt

@@ -0,0 +1,4 @@
+bertviz
+jupyter
+scikit-learn
+faiss-cpu