updated needed files

labeling.py

@@ -1,125 +0,0 @@
-import jsonlines
-import json
-from tqdm import tqdm
-import time
-from openai import error as openai_error
-import pandas as pd
-import openai
-import time
-import tiktoken
-import os
-import glob
-
-GPT_MODEL = 'gpt-3.5-turbo'
-GPT_TOKEN_LIMIT = 1500
-os.environ["OPENAI_API_KEY"] = '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'
-# os.environ["OPENAI_API_KEY"] = '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'
-openai.api_key = os.environ["OPENAI_API_KEY"]
-
-LAST_INDEX_FILE_ADDR = 'last_index.txt'
-TOKEN_COUNT_FILE_ADDR = 'tikitoken_count.txt'
-
-def num_tokens(text: str, model: str = GPT_MODEL) -> int:
-    """Return the number of tokens in a string."""
-    encoding = tiktoken.encoding_for_model(model)
-    return len(encoding.encode(text))
-
-
-def extract_seen_ids():
-    seen_ids = set()
-    for tagged_data_addr in glob.iglob('./tagged_data*'):
-      seen_ids.update([json.loads(line)['id'] for line in open(tagged_data_addr)])
-    return seen_ids
-
-
-def get_keyphrase_by_gpt(document) -> str:
-  global error_count
-  # prompt = 'extract main keywords from below document as sorted list (sort by importance). you should not use numbers for counting them. you should generate less than 10 keywords.'
-  # prompt = 'Output only valid JSON list. Please extract the main keywords from the following document. The keywords should be in a comma-separated list, sorted by their importance. Do not use numbers to count the keywords. Try to generate less than 10 keywords.'
-  prompt = 'there is a popular NLP task named KPE (keyphrase Extraction). please extract keyphrases from below article as a perfect Persian KPE model. '
-  role_prompt  = 'return your answer using json list format'
-  message = prompt + '\n' + document
-  # message = prompt + '\n' + document
-  # message = document
-  messages = [
-      # {"role": "system", "content": "Output only valid JSON list"},
-      {"role": "system", "content": role_prompt},
-      {"role": "user", "content": message},
-  ]
-  try:
-    response = openai.ChatCompletion.create(
-      model=GPT_MODEL,
-      messages=messages,
-      temperature=0
-    )
-    response_message = response["choices"][0]["message"]["content"]
-    error_count = 0
-    return response_message
-  except Exception as e:
-    if error_count > 3:
-        raise e
-    error_count += 1
-    time.sleep(20)
-    return []
-
-#input_data = [json.load(line) for line in open('all_data.json').read().splitlines())
-#input_data = open('all_data.json')
-input_data = pd.read_csv('truncated_wiki_plus_shuffled_41203.csv')
-#print('len input data : ', len(input_data))
-try:
-   last_index = int(open(LAST_INDEX_FILE_ADDR).read())
-   print('load last index: ', last_index)
-except:
-  print('error in loading last index')
-  last_index = 0
-
-
-try:
-   token_count = int(open(TOKEN_COUNT_FILE_ADDR).read())
-   print('load token count: ', token_count)
-except:
-  print('error in loading token_count')
-  token_count = 0
-   
-json_f_writer = jsonlines.open(f'tagged_data.jsonl_{str(last_index)}', mode='w')
-seen_ids = extract_seen_ids()
-for _, row_tup in enumerate(tqdm(input_data.iterrows(),total=len(input_data))):
-    index, row = row_tup
-    text = row['truncated_text_300']
-    id = row['id']
-    
-    #filter by last index
-    if index < last_index:
-       print('skipping index: ', index)
-       continue
-    
-    #filter by seen ids
-    if id in seen_ids:
-       print('repated id and skip')
-       continue
-    
-    #filter by gpt max token
-    text_gpt_token_count = num_tokens(text, model=GPT_MODEL)
-    if  text_gpt_token_count > GPT_TOKEN_LIMIT:
-       continue
-    
-    token_count += text_gpt_token_count
-    keyphrases = get_keyphrase_by_gpt(text)
-    try:
-        keyphrases = json.loads(keyphrases)
-        if type(keyphrases) != list:
-        # if type(keyphrases) == str:
-        #     keyphrases = keyphrases.split(',')
-        # else:
-            print(str(index), ': not a list!')
-    except:
-        print(str(index), ':invalid json!')
-
-    new_train_item = {'id': id, 'keyphrases':keyphrases}
-    json_f_writer.write(new_train_item)
-    last_index_f = open(LAST_INDEX_FILE_ADDR, 'w+')
-    last_index_f.write(str(index))
-    token_count_f = open(TOKEN_COUNT_FILE_ADDR, 'w+')
-    token_count_f.write(str(token_count))
-
-print(token_count)

main.py

@@ -1,71 +0,0 @@
-import uvicorn
-import os
-from typing import Union
-from fastapi import FastAPI
-from kpe import KPE
-from fastapi.middleware.cors import CORSMiddleware
-# from fastapi.middleware.trustedhost import TrustedHostMiddleware
-from fastapi import APIRouter , Query
-from sentence_transformers import SentenceTransformer
-import utils
-from ranker import get_sorted_keywords
-from pydantic import BaseModel
-
-
-app = FastAPI(
-    title="AHD Persian KPE",
-    # version=config.settings.VERSION,
-    description="Keyphrase Extraction",
-    openapi_url="/openapi.json",
-    docs_url="/",
-)
-
-TRAINED_MODEL_ADDR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'trained_model', 'trained_model_10000.pt')
-kpe = KPE(trained_kpe_model= TRAINED_MODEL_ADDR, flair_ner_model='flair/ner-english-ontonotes-large', device='cpu')
-ranker_transformer = SentenceTransformer('paraphrase-multilingual-mpnet-base-v2',  device='cpu')
-# Sets all CORS enabled origins
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],  #str(origin) for origin in config.settings.BACKEND_CORS_ORIGINS
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-
-
-
-class KpeParams(BaseModel):
-    text:str
-    count:int=10000
-    using_ner:bool=True
-    return_sorted:bool=False
-
-
-router = APIRouter()
-
-
-@router.get("/")
-def home():
-    return "Welcome to AHD Keyphrase Extraction Service"
-
-
-@router.post("/extract", description="extract keyphrase from persian documents")
-async def extract(kpe_params: KpeParams):
-    global kpe
-    text = utils.normalize(kpe_params.text)
-    kps = kpe.extract(text, using_ner=kpe_params.using_ner)    
-    if kpe_params.return_sorted:
-        kps = get_sorted_keywords(ranker_transformer, text, kps)
-    else:
-        kps = [(kp, 1) for kp in kps]
-    if len(kps) > kpe_params.count:
-        kps = kps[:kpe_params.count]
-    return kps
-
-
-app.include_router(router)
-
-
-if __name__ == "__main__":
-    uvicorn.run("main:app",host="0.0.0.0", port=7201)

ner_data_construction.py

@@ -1,70 +0,0 @@
-import pandas as pd
-import json
-import glob
-
-
-def tag_document(keywords, tokens):
-
-    # Initialize the tags list with all O's
-    tags = ['O'] * len(tokens)
-
-    # Loop over the keywords and tag the document
-    for keyword in keywords:
-        # Split the keyword into words
-        keyword_words = keyword.split()
-
-        # Loop over the words in the document
-        for i in range(len(tokens)):
-            # If the current word matches the first word of the keyword
-            if tokens[i] == keyword_words[0]:
-                match = True
-                # Check if the rest of the words in the keyword match the following words in the document
-                for j in range(1, len(keyword_words)):
-                    if i+j >= len(tokens) or tokens[i+j] != keyword_words[j]:
-                        match = False
-                        break
-                # If all the words in the keyword match the following words in the document, tag them as B-KEYWORD and I-KEYWORD
-                if match:
-                    tags[i] = 'B-KEYWORD'
-                    for j in range(1, len(keyword_words)):
-                        tags[i+j] = 'I-KEYWORD'
-
-    return tags
-
-
-def create_tner_dataset(all_tags, all_tokens, output_file_addr):
-    output_f = open(output_file_addr, 'a+')
-    for tags, tokens in zip(all_tags, all_tokens):
-        for tag, tok in zip(tags, tokens):
-            line = '\t'.join([tok, tag])
-            output_f.write(line)
-            output_f.write('\n')
-        output_f.write('\n')
-    
-
-if __name__ == '__main__':
-
-    data_df = pd.read_csv('truncated_wiki_plus_shuffled_41203.csv')
-    id2document = data_df.set_index('id')['truncated_text_300'].to_dict()
-
-    
-    #tag documents!
-    print('------------------  tag documents --------------------')
-    all_tags = []
-    all_tokens = []
-    for tagged_data_addr in glob.iglob('./tagged_data*'):
-        for line in open(tagged_data_addr):
-            item = json.loads(line)
-            if type(item['keyphrases']) == list:
-                keywords = item['keyphrases']
-                document = id2document[item['id']]
-                tokens = document.split()
-                tags = tag_document(keywords, tokens)
-                assert len(tokens) == len(tags)
-                all_tags.append(tags)
-                all_tokens.append(tokens)
-                print(len(keywords), len(tags), len(document.split()), len([t for t in tags if t[0]== 'B']))
-    nerda_dataset = {'sentences':all_tokens, 'tags': all_tags}
-    with open('nerda_dataset.json', 'w+') as f:
-        json.dump(nerda_dataset, f)
-    # create_tner_dataset(all_tags, all_tokens, output_file_addr='./sample_train.conll')

predict.py

@@ -1,14 +0,0 @@
-import time
-from kpe import KPE
-import sys
-import os
-
-
-if __name__ == '__main__':
-    TRAINED_MODEL_ADDR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'trained_model_10000.pt')
-    text_addr = sys.argv[1]
-    text = open(text_addr).read()
-    kpe = KPE(trained_kpe_model= TRAINED_MODEL_ADDR, flair_ner_model='flair/ner-english-ontonotes-large', device='cpu')
-    s =time.time()
-    print(kpe.extract(text))
-    print(time.time() - s)