Spaces:
Sleeping
Sleeping
added ingestion functionality
Browse files
ingest.py
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext
|
2 |
+
from llama_index.text_splitter import SentenceSplitter
|
3 |
+
|
4 |
+
documents = SimpleDirectoryReader("./data").load_data()
|
5 |
+
|
6 |
+
text_splitter = SentenceSplitter(chunk_size=512, chunk_overlap=10)
|
7 |
+
service_context = ServiceContext.from_defaults(text_splitter=text_splitter)
|
8 |
+
|
9 |
+
index = VectorStoreIndex.from_documents(
|
10 |
+
documents, service_context=service_context
|
11 |
+
)
|