isayahc commited on
Commit
9580ca5
1 Parent(s): 60c8e5a

added ingestion functionality

Browse files
Files changed (1) hide show
  1. ingest.py +11 -0
ingest.py ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext
2
+ from llama_index.text_splitter import SentenceSplitter
3
+
4
+ documents = SimpleDirectoryReader("./data").load_data()
5
+
6
+ text_splitter = SentenceSplitter(chunk_size=512, chunk_overlap=10)
7
+ service_context = ServiceContext.from_defaults(text_splitter=text_splitter)
8
+
9
+ index = VectorStoreIndex.from_documents(
10
+ documents, service_context=service_context
11
+ )