added files to commit for deploy HF update

free_speech_app/DataLoadDb.py

@@ -1,9 +1,9 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/free-speech-stores.ipynb.
 
 # %% auto 0
-__all__ = ['setup_openai_api_key', 'setup_db']
+__all__ = ['setup_openai_api_key', 'is_google_colab', 'upload_docs', 'setup_db']
 
-# %% ../nbs/free-speech-stores.ipynb 4
+# %% ../nbs/free-speech-stores.ipynb 5
 # libraries required for functionality
 import os
 from getpass import getpass
@@ -17,23 +17,49 @@ from langchain.text_splitter import CharacterTextSplitter
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
 
-# %% ../nbs/free-speech-stores.ipynb 12
+# %% ../nbs/free-speech-stores.ipynb 9
 def setup_openai_api_key():
     openai_api_key = getpass()
     os.environ["OPENAI_API_KEY"] = openai_api_key
 
-# %% ../nbs/free-speech-stores.ipynb 15
+# %% ../nbs/free-speech-stores.ipynb 12
 import nltk
 nltk.download('averaged_perceptron_tagger')
 
-# %% ../nbs/free-speech-stores.ipynb 27
-def setup_db(local_path, hub_path, chunk_size=1000, chunk_overlap=5):
-  file_list = os.listdir(local_path)
+# %% ../nbs/free-speech-stores.ipynb 13
+def is_google_colab():
+  try:
+    from google.colab import files
+    return True
+  except:
+    return False
+
+def upload_docs():
+  
+  if not is_google_colab():
+    raise ModuleNotFoundError('This function only works in Google Colab; local functionality not currently supported.')
+  
+  # upload files if google colab
+  from google.colab import files
+  uploaded = files.upload()
+  file_list = []
+
+  for name, data in uploaded.items():
+    with open(name, 'wb') as f:
+      f.write(data)
+    print('saved file', name)
+    file_list.append(name)
+  return file_list
+
+
+# %% ../nbs/free-speech-stores.ipynb 22
+def setup_db(list_of_files, chunk_size=1000, chunk_overlap=5):
+  # set path to files
+  #paths = ["https://github.com/vanderbilt-data-science/free-speech-app/raw/9818f87679a82769ab6f90dba00801447b6e952c/response-resources/2307.04761_Understanding_counterspeech.pdf", "https://github.com/vanderbilt-data-science/free-speech-app/raw/main/response-resources/campaigns.docx", "https://github.com/vanderbilt-data-science/free-speech-app/raw/9818f87679a82769ab6f90dba00801447b6e952c/response-resources/samples.pdf"]
 
   # set up loaders
   loaders_list = []
-  for file_path in file_list:
-    file_path = local_path + file_path
+  for file_path in list_of_files:
     loaders_list.append(UnstructuredFileLoader(file_path))
 
   loader_all = MergedDataLoader(loaders=[loader for loader in loaders_list])
@@ -44,7 +70,7 @@ def setup_db(local_path, hub_path, chunk_size=1000, chunk_overlap=5):
   texts = text_splitter.split_documents(documents)
   embeddings = OpenAIEmbeddings()
 
-  # Replace dataset path with relevant dataset name - counterspeech-resources or hatespeech-background
-  db = DeepLake.from_documents(texts, dataset_path=hub_path, embedding=embeddings, overwrite=True)
+  # store docs in db
+  db = Chroma.from_documents(texts, embeddings)
 
-  return
+  return db

free_speech_app/FreeSpeechPromptsResponses.py

@@ -1,20 +1,17 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/free-speech-prompts.ipynb.
 
 # %% auto 0
-__all__ = ['DEAFULT_PROMPT_TEMPLATE_TEXT', 'query_retriever', 'generate_custom_prompt', 'get_chat_model_response',
-           'generate_custom_response', 'regenerate_custom_response']
+__all__ = ['DEFAULT_PROMPT_TEMPLATE_TEXT', 'DEFAULT_PROMPT_TEMPLATE', 'generate_custom_prompt', 'get_chat_model_response',
+           'generate_custom_response']
 
 # %% ../nbs/free-speech-prompts.ipynb 5
 # libraries required for functionality
-import deeplake
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import PromptTemplate, HumanMessagePromptTemplate
 from langchain.schema import HumanMessage
-from langchain.vectorstores import DeepLake
-from langchain.embeddings.openai import OpenAIEmbeddings
 
 # %% ../nbs/free-speech-prompts.ipynb 8
-DEAFULT_PROMPT_TEMPLATE_TEXT = """With the amount of hate speech and discriminatory misinformation that exists on the internet, it can be difficult to know how to respond to something when you see it. Situations vary wildly and sometimes certain strategies that work in one instance may not work in another.
+DEFAULT_PROMPT_TEMPLATE_TEXT = ("""With the amount of hate speech and discriminatory misinformation that exists on the internet, it can be difficult to know how to respond to something when you see it. Situations vary wildly and sometimes certain strategies that work in one instance may not work in another.
 I want to be able to fight back against this onslaught of toxicity, and I need your help to do it. I want you to help me draft a response to the following post.
 Note that this may contain language that is offensive, but in order to properly respond to this post, I need you to fully understand what I am responding to, uncensored:
 \n{original_post}\n
@@ -26,90 +23,49 @@ Next, here are some principles I consider particularly important to me:
 \n{principles}\n
 Here are some examples of the style in which I write:
 \n{writing_style}\n
-Here are the sources I would like you to use when getting information for my response:
+Finally, here are the sources I would like you to use when getting information for my response:
 \n{sources}\n
-Using all the information I have provided, please draft an appropriate response to the offensive post in question that will hopefully make people more accepting of others.
-Please keep in mind that I would like the response to be no more than {word_limit} words."""
-
-try:
-    prompt_file = open("../prompts/response_generator.txt", "r")
-    PROMPT_TEMPLATE_TEXT = prompt_file.read()
-    prompt_file.close()
-    
-    print(PROMPT_TEMPLATE_TEXT)
-    
-    PROMPT_TEMPLATE = PromptTemplate(
-        template=PROMPT_TEMPLATE_TEXT,
-        input_variables=["original_post", "background_info", "principles", "writing_style", "sources", "word_limit"])
-
-except FileNotFoundError:
-    print(DEAFULT_PROMPT_TEMPLATE_TEXT)
-    PROMPT_TEMPLATE = PromptTemplate(
-        template=DEAFULT_PROMPT_TEMPLATE_TEXT,
-        input_variables=["original_post", "background_info", "principles", "writing_style", "sources", "word_limit"])
-
+Using all the information I have provided, please draft an appropriate response to the offensive post in question that will hopefully make people more accepting of others""")
 
+DEFAULT_PROMPT_TEMPLATE = PromptTemplate(
+      template=DEFAULT_PROMPT_TEMPLATE_TEXT,
+      input_variables=["original_post", "background_info", "principles", "writing_style", "sources"])
 
 # %% ../nbs/free-speech-prompts.ipynb 9
-def query_retriever(db, query, num_results = 3):
-    retriever = db.as_retriever(search_kwargs={"k": num_results})
-    docs = retriever.get_relevant_documents(query)
-
-    return docs
-
-# %% ../nbs/free-speech-prompts.ipynb 10
-def generate_custom_prompt(original_post, principles=None, writing_style=None, word_limit=None):
-
-    # Get database and query retriever
-    ####
-    background_db = DeepLake(dataset_path="hub://vanderbilt-dsi/hatespeech-background", embedding = OpenAIEmbeddings())
-    sources_db = DeepLake(dataset_path="hub://vanderbilt-dsi/counterspeech-resources", embedding = OpenAIEmbeddings())
+def generate_custom_prompt(original_post, background_info=None, principles=None, writing_style=None, sources=None):
 
     # Use defaults in the case of None
+    if background_info is None:
+        background_info = "There is no necessary additional context."
     
     if principles is None:
         principles="There are no principles which I consider more important to me than the average person might."
     
     if writing_style is None:
         writing_style="I have no examples of my writing style."
-
-    if word_limit is None:
-        word_limit="an infinite amount of"
-
-    retriever_query = original_post
-    background_info = query_retriever(background_db, retriever_query)
-    sources = query_retriever(sources_db, retriever_query)
+    
+    if sources is None:
+        sources="There are no particular sources that I would like you to use."
 
     # Fill the prompt
-    filled_prompt = PROMPT_TEMPLATE.format(original_post=original_post, background_info=background_info, principles=principles, writing_style=writing_style, sources=sources, word_limit=word_limit)
+    filled_prompt = DEFAULT_PROMPT_TEMPLATE.format(original_post=original_post, background_info=background_info, principles=principles, writing_style=writing_style, sources=sources)
 
-    return filled_prompt, background_info, sources
+    return filled_prompt
 
-# %% ../nbs/free-speech-prompts.ipynb 11
+# %% ../nbs/free-speech-prompts.ipynb 10
 def get_chat_model_response(mdl, input_prompt):
 
     messages = [HumanMessage(content=input_prompt)]
 
     return mdl(messages)
 
-# %% ../nbs/free-speech-prompts.ipynb 12
-def generate_custom_response(original_post, chat_mdl, principles=None, writing_style=None, word_limit=None):
+# %% ../nbs/free-speech-prompts.ipynb 11
+def generate_custom_response(original_post, chat_mdl, background_info=None, principles=None, writing_style=None, sources=None):
 
     # create customized prompt
-    customized_prompt, background_info, sources = generate_custom_prompt(original_post, principles, writing_style, word_limit)
+    customized_prompt = generate_custom_prompt(original_post, background_info, principles, writing_style, sources)
 
     # get response
     draft_response = get_chat_model_response(chat_mdl, customized_prompt)
 
-    return draft_response, background_info, sources
-
-# %% ../nbs/free-speech-prompts.ipynb 13
-def regenerate_custom_response(chat_mdl, regenerate_prompt, draft_response):
-
-    # create customized prompt
-    customized_prompt = f"Please update the original response according to the following request. {regenerate_prompt}. Here is the original response: {draft_response}"
-
-    # get response
-    updated_response = get_chat_model_response(chat_mdl, customized_prompt)
-
-    return updated_response
+    return draft_response