Upload 3 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+linking_df_technical_min.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,121 @@
+import pandas as pd
+from utils import normalize_text
+import streamlit as st
+
+# load wikipedia data
+wiki_df = pd.read_csv("../knowledge_platform/wiki_output/kensho_en_wiki_typing_technical.csv")
+
+# filter out technical articles
+exclude_ids = set(wiki_df[(wiki_df.exclude == True) | (wiki_df.technical == False)].page_id.to_list())
+include_skpes = set(wiki_df[wiki_df.page_id.apply(lambda x: x not in exclude_ids)].skpe_id.to_list())
+
+wiki_df = wiki_df.drop(columns=['Unnamed: 0', 'en_probs', 'exclude'])
+wiki_df = wiki_df.rename(columns={'title_x': 'en_title'})
+
+# load kg df
+
+"""Load wikidata"""
+
+wikidata_df = pd.read_csv("../knowledge_platform/kg_data/wikidata_ss_processed.csv")
+
+# filter technical wikidata
+wikidata_df = wikidata_df[wikidata_df.apply(lambda x: x.source_skpe in include_skpes and x.target_skpe in include_skpes, axis=1)]
+
+
+"""KG Infer data"""
+
+
+rebel_infer_df = pd.read_csv("../knowledge_platform/kg_data/rebel_inference_processed_ss.csv")
+
+# filter technical
+rebel_infer_df = rebel_infer_df[rebel_infer_df.apply(lambda x: type(x.source_skpe_id) == str and type(x.target_skpe_id) == str, axis=1)]
+
+rebel_infer_df = rebel_infer_df.drop(columns=['instance_id', 'source_text', 'target_text'])
+rebel_infer_df = rebel_infer_df.rename(columns={'source_skpe_id': 'source_skpe', 'target_skpe_id': 'target_skpe', 'source': 'source_en', 'target': 'target_en'})
+
+
+wikidata_df['source'] = 'wikidata'
+rebel_infer_df['source'] = 'rebel_wikipedia'
+
+rebel_infer_df = rebel_infer_df[rebel_infer_df.source_skpe != rebel_infer_df.target_skpe]
+
+kg_df = pd.concat([wikidata_df, rebel_infer_df])
+
+
+# ???
+
+
+# load entity linking dictionary
+linking_df = pd.read_csv('./linking_df_technical_min.csv')
+
+# User Input
+input_text = st.text_input(
+    label="Enter first entity name",
+    value="semiconductor",
+    key="ent",
+)
+
+# normalise and match
+text_norm = normalize_text(input_text)
+match_df = linking_df[linking_df.text == text_norm]
+
+# top match skpe
+if len(match_df) > 0:
+
+    top_skpe = match_df.skpe_id.mode()[0]
+    all_skpe = set(match_df.skpe_id.to_list())
+    skpe_to_count = dict(match_df.skpe_id.value_counts())
+
+    # Match list
+    wiki_match_df = wiki_df[wiki_df.skpe_id.apply(lambda x: x in all_skpe)].copy()
+    wiki_match_df['link_score'] = wiki_match_df['skpe_id'].apply(lambda x: skpe_to_count[x] / sum(skpe_to_count.values()))
+    wiki_match_df = wiki_match_df.sort_values(by='link_score', ascending=False)
+    
+else:
+    st.write("no matches")
+
+# show similar results
+wiki_match_df.sort_values(by='views', ascending=False)[:5]
+
+# Stuff that are made out of input
+made_of_df = kg_df[(kg_df.relation == 'made_from_material') & (kg_df.target_skpe == top_skpe)].copy()
+# made_of_list = made_of_df.source_skpe.to_list()
+
+all_paths = []
+
+
+# iterate over first rows
+for first_edge in made_of_df.itertuples():
+    
+    first_item = first_edge.source_skpe
+
+    # applications of stuff made out of first item
+    use_df = kg_df[((kg_df.relation == 'has_use') & (kg_df.source_skpe == first_item)) | ((kg_df.relation == 'uses') & (kg_df.target_skpe == first_item))]
+
+    # add all 2 len paths
+    for second_edge in use_df.itertuples():
+        all_paths.append([first_edge, second_edge])
+        
+    # expand to part of
+    
+    # applications of stuff made out of steel # 1
+    part_df = kg_df[((kg_df.relation == 'has_part') & (kg_df.target_skpe == first_item)) | (kg_df.relation == 'part_of') & (kg_df.source_skpe == first_item)]
+
+    # iterate over all parts of product
+    for second_edge in part_df.itertuples():
+
+        # select second item
+        second_item = second_edge.source_skpe if second_edge.relation == 'has_part' else second_edge.target_skpe
+
+        # get uses of second item
+        use_df = kg_df[((kg_df.relation == 'has_use') & (kg_df.source_skpe == second_item)) | ((kg_df.relation == 'uses') & (kg_df.target_skpe == second_item))]
+
+        # add all 3 len paths
+        for third_edge in use_df.itertuples():
+            all_paths.append([first_edge, second_edge, third_edge])
+
+# print all paths
+for path in all_paths:
+    for edge in path:
+        st.write(f"{edge.source_en} --{edge.relation}--> {edge.target_en} | source: {edge.source}")
+    st.write("------")

linking_df_technical_min.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37a47468ddb08e8c54b8b55d9b488c84359a366ad6c1763b84587cea9afdbaa2
+size 157547217

utils.py

@@ -0,0 +1,50 @@
+import json
+
+def dump_json(file, path):
+    """Save json object"""
+
+    with open(path, 'w', encoding='utf-8') as f:
+        json.dump(file, f, indent=4, ensure_ascii=False)
+    print("Saved json to path: " + str(path))
+
+def load_json(path):
+    """load json object"""
+    with open(path, 'rb') as f:
+        data = json.load(f)
+    print("Loaded json from path: " + str(path))
+    return data
+
+def lead_k_sentences(text, k=50):
+    """Select the first k sentences from a Japanese document"""
+
+    DELIMITER = '。'
+
+    if DELIMITER in text:
+        segments = [seg for seg in text.split(DELIMITER)[:k] if len(seg) > 0]
+        return DELIMITER.join(segments) + DELIMITER
+    else:
+        return text
+    
+import jsonlines
+
+def read_jsonlines(path):    
+    with jsonlines.open(path) as reader:
+        lines = [obj for obj in reader]
+    return lines
+
+def write_jsonlines(file, path):
+    with jsonlines.open(path, 'w') as writer:
+        writer.write_all(file)
+        
+import re
+
+def normalize_text(s):
+    s = str(s)
+    # if not s.isupper():
+    #     s = re.sub(r"(\w)([A-Z])", r"\1 \2", s) # Respace
+    s = re.sub(r'\（.*\）', '', s) # Remove japanese brackets
+    s = re.sub(r'\(.*\)', '', s) # Remove english brackets
+    s = s.strip()
+    s = s.replace(' ', '_')
+    s = s.upper()
+    return s