import http.client as http_client
import json
import logging
import os
import pprint
import re
import string
import gradio as gr
import requests
pp = pprint.PrettyPrinter(indent=2)
def get_docid_html(docid):
data_org, dataset, docid = docid.split("/")
docid_html = """{}/{}""".format(
dataset, data_org + "/" + dataset, docid
)
return docid_html
PII_TAGS = {"KEY", "EMAIL", "USER", "IP_ADDRESS", "ID", "IPv4", "IPv6"}
PII_PREFIX = "PI:"
def process_pii(text):
for tag in PII_TAGS:
text = text.replace(
PII_PREFIX + tag,
"""REDACTED {}""".format(tag),
)
return text
def process_meta_roots(result):
meta_html = (
"""
{}
""".format(
result["meta"]["url"], result["meta"]["url"]
)
if "meta" in result and result["meta"] is not None and "url" in result["meta"]
else ""
)
"""
'meta': { 'docs': [ { 'TEXT': 'Hello World Example Hello '
'World Page Hello World.',
'URL': 'http://images.slideplayer.com/8/2335183/slides/slide_6.jpg',
'_id': 592573973},
{ 'TEXT': 'Hello World Example Hello '
'World Page Hello World.',
'URL': 'http://images.slideplayer.com/8/2335183/slides/slide_9.jpg',
'_id': 1807595732},
{ 'TEXT': 'Hello World Example Hello '
'World Page Hello World.',
'URL': 'http://images.slideplayer.com/8/2335183/slides/slide_10.jpg',
'_id': 1864921031},
{ 'TEXT': 'Hello World Example Hello '
'World Page Hello World!',
'URL': 'http://images.slideplayer.com/8/2335183/slides/slide_5.jpg',
'_id': 1964462104},
{ 'TEXT': 'Hello World Example Hello '
'World Page Hello World.',
'URL': 'http://images.slideplayer.com/8/2335183/slides/slide_8.jpg',
'_id': 2167992166}]},
"""
def process_meta_laion(result):
meta_html = """"""
if "meta" not in result:
return meta_html
for doc in result["meta"]["docs"]:
# doc = json.loads(doc)
print(type(doc), doc)
print(doc["URL"])
meta_html += """
{}
""".format(doc["URL"], doc["URL"])
return meta_html
def process_results(results, highlight_terms):
if len(results) == 0:
return """
No results retrieved.
"""
results_html = ""
for result in results:
tokens = result["text"].split()
tokens_html = []
for token in tokens:
if token in highlight_terms:
tokens_html.append("{}".format(token))
else:
tokens_html.append(token)
tokens_html = " ".join(tokens_html)
tokens_html = process_pii(tokens_html)
meta_html = process_meta_laion(result)
docid_html = """Document ID: {}
""".format(result["docid"]) # get_docid_html(result["docid"])
language_html = """Language: {}
""".format(result["lang"])
results_html += tokens_html + meta_html + "
"
return results_html + "
"
def scisearch(query, language, num_results=10):
try:
query = query.strip()
if query == "" or query is None:
return
post_data = {"query": query, "k": num_results}
if language != "detect_language":
post_data["lang"] = language
output = requests.post(
os.environ.get("address"),
headers={"Content-type": "application/json"},
data=json.dumps(post_data),
timeout=60,
)
payload = json.loads(output.text)
pp.pprint(payload)
if "err" in payload:
if payload["err"]["type"] == "unsupported_lang":
detected_lang = payload["err"]["meta"]["detected_lang"]
return f"""
Detected language {detected_lang} is not supported.
Please choose a language from the dropdown or type another query.
"""
results = payload["results"]
highlight_terms = payload["highlight_terms"]
if language == "detect_language":
return (
(
f"""
Detected language: {results[0]["lang"]}
"""
if len(results) > 0 and language == "detect_language"
else ""
)
+ process_results(results, highlight_terms)
)
if language == "all":
results_html = ""
for lang, results_for_lang in results.items():
if len(results_for_lang) == 0:
results_html += f"""
No results for language: {lang}
"""
continue
collapsible_results = f"""
Results for language: {lang}
{process_results(results_for_lang, highlight_terms)}
"""
results_html += collapsible_results
return results_html
return process_results(results, highlight_terms)
except Exception as e:
results_html = f"""
Raised {type(e).__name__}
Check if a relevant discussion already exists in the Community tab. If not, please open a discussion.
"""
print(e)
return results_html
def flag(query, language, num_results, issue_description):
try:
post_data = {"query": query, "k": num_results, "flag": True, "description": issue_description}
if language != "detect_language":
post_data["lang"] = language
output = requests.post(
os.environ.get("address"),
headers={"Content-type": "application/json"},
data=json.dumps(post_data),
timeout=120,
)
results = json.loads(output.text)
except:
print("Error flagging")
return ""
description = """# GAIA 🌖🌏
A large scale text corpora search engine."""
if __name__ == "__main__":
demo = gr.Blocks(
css=".underline-on-hover:hover { text-decoration: underline; } .flagging { font-size:12px; color:Silver; }"
)
with demo:
with gr.Row():
gr.Markdown(value=description)
with gr.Row():
query = gr.Textbox(lines=2, placeholder="Type your query here...", label="Query")
with gr.Row():
lang = gr.Dropdown(
choices=[
"ar",
"ca",
"code",
"en",
"es",
"eu",
"fr",
"id",
"indic",
"nigercongo",
"pt",
"vi",
"zh",
"detect_language",
"all",
],
value="en",
label="Language",
)
with gr.Row():
k = gr.Slider(1, 100, value=10, step=1, label="Max Results")
with gr.Row():
submit_btn = gr.Button("Submit")
with gr.Row():
results = gr.HTML(label="Results")
flag_description = """
If you choose to flag your search, we will save the query, language and the number of results you requested.
Please consider adding any additional context in the box on the right.
"""
with gr.Column(visible=False) as flagging_form:
flag_txt = gr.Textbox(
lines=1,
placeholder="Type here...",
label="""If you choose to flag your search, we will save the query, language and the number of results
you requested. Please consider adding relevant additional context below:""",
)
flag_btn = gr.Button("Flag Results")
flag_btn.click(flag, inputs=[query, lang, k, flag_txt], outputs=[flag_txt])
def submit(query, lang, k):
if query == "":
return ["", ""]
return {
results: scisearch(query, lang, k),
flagging_form: gr.update(visible=True),
}
submit_btn.click(submit, inputs=[query, lang, k], outputs=[results, flagging_form])
demo.launch(enable_queue=True, debug=True)