Spaces:
Running
Running
File size: 2,554 Bytes
3c87692 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
"""Load html from files, clean up, split, ingest into Weaviate."""
import os
from pathlib import Path
import weaviate
from bs4 import BeautifulSoup
from langchain.text_splitter import CharacterTextSplitter
def clean_data(data):
soup = BeautifulSoup(data)
text = soup.find_all("main", {"id": "main-content"})[0].get_text()
return "\n".join([t for t in text.split("\n") if t])
docs = []
metadatas = []
for p in Path("langchain.readthedocs.io/en/latest/").rglob("*"):
if p.is_dir():
continue
with open(p) as f:
docs.append(clean_data(f.read()))
metadatas.append({"source": p})
text_splitter = CharacterTextSplitter(
separator="\n",
chunk_size=1000,
chunk_overlap=200,
length_function=len,
)
documents = text_splitter.create_documents(docs, metadatas=metadatas)
WEAVIATE_URL = os.environ["WEAVIATE_URL"]
client = weaviate.Client(
url=WEAVIATE_URL,
additional_headers={"X-OpenAI-Api-Key": os.environ["OPENAI_API_KEY"]},
)
client.schema.delete_class("Paragraph")
client.schema.get()
schema = {
"classes": [
{
"class": "Paragraph",
"description": "A written paragraph",
"vectorizer": "text2vec-openai",
"moduleConfig": {
"text2vec-openai": {
"model": "ada",
"modelVersion": "002",
"type": "text",
}
},
"properties": [
{
"dataType": ["text"],
"description": "The content of the paragraph",
"moduleConfig": {
"text2vec-openai": {
"skip": False,
"vectorizePropertyName": False,
}
},
"name": "content",
},
{
"dataType": ["text"],
"description": "The link",
"moduleConfig": {
"text2vec-openai": {
"skip": True,
"vectorizePropertyName": False,
}
},
"name": "source",
},
],
},
]
}
client.schema.create(schema)
with client.batch as batch:
for text in documents:
batch.add_data_object(
{"content": text.page_content, "source": str(text.metadata["source"])},
"Paragraph",
)
|