Spaces:
Runtime error
Runtime error
| import langchain.document_loaders | |
| from langchain.document_loaders import DirectoryLoader, PyPDFLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| from langchain.schema import Document | |
| from langchain.embeddings import OpenAIEmbeddings | |
| from langchain.vectorstores.chroma import Chroma | |
| import os | |
| import shutil | |
| from langchain.vectorstores.chroma import Chroma | |
| from langchain.embeddings import OpenAIEmbeddings | |
| from langchain.chat_models import ChatOpenAI | |
| from langchain.prompts import ChatPromptTemplate | |
| def get_chunks(file_path): | |
| loader = PyPDFLoader(file_path) | |
| documents = loader.load() | |
| text_splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=300, | |
| chunk_overlap=100, | |
| length_function=len, | |
| add_start_index=True, | |
| ) | |
| chunks = text_splitter.split_documents(documents) | |
| return chunks | |
| def get_vectordb(chunks): | |
| # CHROMA_PATH = f"../../chroma/{CHROMA_PATH}" | |
| db = Chroma.from_documents(chunks, embedding_function=OpenAIEmbeddings()) | |
| # if os.path.exists(CHROMA_PATH): | |
| # db = Chroma(persist_directory=CHROMA_PATH, embedding_function=OpenAIEmbeddings()) | |
| # else: | |
| # db = Chroma.from_documents( | |
| # chunks, OpenAIEmbeddings(), persist_directory=CHROMA_PATH | |
| # ) | |
| # db.persist() | |
| # print(f"Saved {len(chunks)} chunks to {CHROMA_PATH}.") | |
| return db | |
| def gen_sample(text, db): | |
| PROMPT_TEMPLATE = """ | |
| Answer the question based only on the following context: | |
| {context} | |
| --- | |
| Answer the question based on the above context: {question} | |
| """ | |
| query_text = f""" | |
| Act as the author of a Choose Your Own Adventure Book. This book is special as it is based on existing material. | |
| Now, as with any choose your own adventure book, you'll have to generate decision paths at certain points in the story. | |
| Your job is to generate 4 decision paths for the given point in the story, if applicable to that point in the story. | |
| If the given part of the story doesn't contain any decisions from which to generate decision paths, don't | |
| generate any. If the given part of the story contains a decision, generate 4 decision paths for that decision. | |
| One among the 4 decision paths should be the original path, the other 3 should deviate from the original path in a sensible manner. | |
| The decision paths should be generated in a way that they are coherent with the existing story. | |
| The result should be a JSON object with the following keys: [text, paths] | |
| text: The given text | |
| paths: The generated decision paths as strings in a list | |
| ```{text}``` | |
| """ | |
| results = db.similarity_search_with_relevance_scores(query_text, k=5) | |
| context_text = "\n\n---\n\n".join([doc.page_content for doc, _score in results]) | |
| prompt_template = ChatPromptTemplate.from_template(PROMPT_TEMPLATE) | |
| prompt = prompt_template.format(context=context_text, question=query_text) | |
| model = ChatOpenAI() | |
| response_text = model.predict(prompt) | |
| return eval(response_text) | |