Update utils.py
Browse files
utils.py
CHANGED
@@ -397,28 +397,40 @@ def document_storage_chroma(splits):
|
|
397 |
|
398 |
########################################################
|
399 |
#Splits für den Vektorstore speichern - bzw. laden
|
400 |
-
def save_splits(
|
401 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
402 |
if not os.path.exists(directory):
|
403 |
os.makedirs(directory)
|
404 |
|
405 |
-
# Speichern der Splits
|
406 |
-
|
407 |
-
with open(
|
408 |
-
pickle.dump(
|
409 |
|
410 |
-
#
|
411 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
412 |
|
413 |
-
def load_splits(directory="chroma/kkg",
|
414 |
# Vollständigen Pfad zur Datei erstellen
|
415 |
-
|
|
|
416 |
|
417 |
-
# Laden der Splits aus der Datei
|
418 |
-
if os.path.exists(
|
419 |
-
with open(
|
420 |
-
|
421 |
-
|
|
|
|
|
|
|
|
|
|
|
422 |
|
423 |
########################################################
|
424 |
#Vektorstore speichern - bzw. laden
|
|
|
397 |
|
398 |
########################################################
|
399 |
#Splits für den Vektorstore speichern - bzw. laden
|
400 |
+
def save_splits(preprocessed_splits, original_splits, directory="chroma/kkg", preprocessed_filename="preprocessed_splits.pkl", original_filename="original_splits.pkl"):
|
401 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
402 |
if not os.path.exists(directory):
|
403 |
os.makedirs(directory)
|
404 |
|
405 |
+
# Speichern der vorverarbeiteten Splits
|
406 |
+
preprocessed_filepath = os.path.join(directory, preprocessed_filename)
|
407 |
+
with open(preprocessed_filepath, "wb") as f:
|
408 |
+
pickle.dump(preprocessed_splits, f)
|
409 |
|
410 |
+
# Speichern der Originalsplits
|
411 |
+
original_filepath = os.path.join(directory, original_filename)
|
412 |
+
with open(original_filepath, "wb") as f:
|
413 |
+
pickle.dump(original_splits, f)
|
414 |
+
|
415 |
+
# Hochladen der Splits-Dateien zum Hugging Face Space
|
416 |
+
upload_file_to_huggingface(preprocessed_filepath, f"{directory}/{preprocessed_filename}")
|
417 |
+
upload_file_to_huggingface(original_filepath, f"{directory}/{original_filename}")
|
418 |
|
419 |
+
def load_splits(directory="chroma/kkg", preprocessed_filename="preprocessed_splits.pkl", original_filename="original_splits.pkl"):
|
420 |
# Vollständigen Pfad zur Datei erstellen
|
421 |
+
preprocessed_filepath = os.path.join(directory, preprocessed_filename)
|
422 |
+
original_filepath = os.path.join(directory, original_filename)
|
423 |
|
424 |
+
# Laden der vorverarbeiteten Splits aus der Datei
|
425 |
+
if os.path.exists(preprocessed_filepath) and os.path.exists(original_filepath):
|
426 |
+
with open(preprocessed_filepath, "rb") as f:
|
427 |
+
preprocessed_splits = pickle.load(f)
|
428 |
+
|
429 |
+
with open(original_filepath, "rb") as f:
|
430 |
+
original_splits = pickle.load(f)
|
431 |
+
|
432 |
+
return preprocessed_splits, original_splits
|
433 |
+
return None, None
|
434 |
|
435 |
########################################################
|
436 |
#Vektorstore speichern - bzw. laden
|