adding new requirements

app.py

@@ -16,30 +16,30 @@ st.markdown("Translation page 🔠")
 # models = {
 #     "Version ✌️": {
 #         "French ➡️ Wolof": {
-#             "checkpoints": "wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_fw_v4",
-#             "tokenizer": "wolof-translate/wolof_translate/tokenizers/t5_tokenizers/tokenizer_v4.json",
+#             "checkpoints": "wolof_translate/checkpoints/t5_small_custom_train_results_fw_v4",
+#             "tokenizer": "wolof_translate/tokenizers/t5_tokenizers/tokenizer_v4.json",
 #             "max_len": None
 #         }
 #     },
 #     "Version ☝️": {
 #         "French ➡️ Wolof": {
-#             "checkpoints": "wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3",
-#             "tokenizer": "wolof-translate/wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json",
+#             "checkpoints": "wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3",
+#             "tokenizer": "wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json",
 #             "max_len": 51
 #             },
 #         "Wolof ➡️ French": {
-#             "checkpoints": "wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3",
-#             "tokenizer": "wolof-translate/wolof_translate/trokenizers/t5_tokenizers/tokenizer_v3.json",
+#             "checkpoints": "wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3",
+#             "tokenizer": "wolof_translate/trokenizers/t5_tokenizers/tokenizer_v3.json",
 #             "max_len": 51
 #         }
 #     }
 # }
 
 # # add special characters from Wolof
-# sp_wolof_chars = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/wolof_special_chars.csv')
+# sp_wolof_chars = pd.read_csv('wolof_translate/data/wolof_writing/wolof_special_chars.csv')
 
 # # add definitions
-# sp_wolof_words = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/definitions.csv')
+# sp_wolof_words = pd.read_csv('wolof_translate/data/wolof_writing/definitions.csv')
 
 # # let us add a callback functions to change the input text
 # def add_symbol_to_text():
@@ -78,10 +78,10 @@ st.markdown("Translation page 🔠")
 #     def get_modelfw_v3():
         
 #         # recuperate checkpoints
-#         checkpoints = torch.load(os.path.join('wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3', "best_checkpoints.pth"), map_location=torch.device('cpu'))
+#         checkpoints = torch.load(os.path.join('wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3', "best_checkpoints.pth"), map_location=torch.device('cpu'))
         
 #         # recuperate the tokenizer
-#         tokenizer_file = "wolof-translate/wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json"
+#         tokenizer_file = "wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json"
         
 #         # initialize the tokenizer
 #         tokenizer = T5TokenizerFast(tokenizer_file=tokenizer_file)
@@ -99,10 +99,10 @@ st.markdown("Translation page 🔠")
 #     def get_modelwf_v3():
         
 #         # recuperate checkpoints
-#         checkpoints = torch.load(os.path.join('wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3', "best_checkpoints.pth"), map_location=torch.device('cpu'))
+#         checkpoints = torch.load(os.path.join('wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3', "best_checkpoints.pth"), map_location=torch.device('cpu'))
         
 #         # recuperate the tokenizer
-#         tokenizer_file = "wolof-translate/wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json"
+#         tokenizer_file = "wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json"
         
 #         # initialize the tokenizer
 #         tokenizer = T5TokenizerFast(tokenizer_file=tokenizer_file)

pages/provide_sentences.py

@@ -4,20 +4,20 @@ import pandas as pd
 st.markdown("Provide your own 🤗 sentences")
 
 # recuperate the already saved sentences (for the moment french/wolof)
-sentences_ = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')
+sentences_ = pd.read_csv('wolof_translate/data/sentences/wolof_french.csv')
 
 sentences = sentences_.copy()
 
 # get french and wolof sentences
-french_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/french.csv')
+french_examples = pd.read_csv('wolof_translate/data/sentences/french.csv')
 
-wolof_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_2.csv')
+wolof_examples = pd.read_csv('wolof_translate/data/sentences/wolof_2.csv')
 
 # add special characters from Wolof
-sp_wolof_chars = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/wolof_special_chars.csv')
+sp_wolof_chars = pd.read_csv('wolof_translate/data/wolof_writing/wolof_special_chars.csv')
 
 # add definitions
-sp_wolof_words = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/definitions.csv')
+sp_wolof_words = pd.read_csv('wolof_translate/data/wolof_writing/definitions.csv')
 
 sp_wolof_words.sort_values(by = ['french', 'wolof'], inplace = True)
 
@@ -89,10 +89,10 @@ def add_new_sentences():
             sentences = pd.DataFrame({sentences.columns.tolist()[0]: [sentence_1], 
                                                    sentences.columns.tolist()[1]: [sentence_2]})
         ## save the result
-        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        sentences.to_csv('wolof_translate/data/sentences/wolof_french.csv', index=False)
         
         # recuperate the already saved sentences (for the moment french/wolof)
-        sentences = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')
+        sentences = pd.read_csv('wolof_translate/data/sentences/wolof_french.csv')
         
         # add the last position to delete and modify
         st.session_state.line1 = len(sentences) - 1
@@ -124,10 +124,10 @@ def add_new_sentences2():
             sentences = pd.DataFrame({sentences.columns.tolist()[0]: [sentence_1], 
                                                    sentences.columns.tolist()[1]: [sentence_2]})
         ## save the result
-        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        sentences.to_csv('wolof_translate/data/sentences/wolof_french.csv', index=False)
         
         # recuperate the already saved sentences (for the moment french/wolof)
-        sentences = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')
+        sentences = pd.read_csv('wolof_translate/data/sentences/wolof_french.csv')
         
         # clean the inputs' contents
         st.session_state.left_sentence = ''
@@ -148,7 +148,7 @@ def delete_line():
     
     else:
         
-        deleted = pd.read_csv('wolof-translate/wolof_translate/data/sentences/deleted_lines.csv')
+        deleted = pd.read_csv('wolof_translate/data/sentences/deleted_lines.csv')
         
         if not deleted.empty:
             
@@ -163,9 +163,9 @@ def delete_line():
         sentences.drop(index = number, inplace=True)
         
         ## save the result
-        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        sentences.to_csv('wolof_translate/data/sentences/wolof_french.csv', index=False)
         
-        deleted.to_csv('wolof-translate/wolof_translate/data/sentences/deleted_lines.csv', index=False)
+        deleted.to_csv('wolof_translate/data/sentences/deleted_lines.csv', index=False)
 
 def modify_line():
     
@@ -177,7 +177,7 @@ def modify_line():
     
     else:
         
-        modified = pd.read_csv('wolof-translate/wolof_translate/data/sentences/modified_lines.csv')
+        modified = pd.read_csv('wolof_translate/data/sentences/modified_lines.csv')
         
         if not modified.empty:
             
@@ -196,9 +196,9 @@ def modify_line():
         sentences.drop(index = number, inplace=True)
         
         ## save the result
-        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        sentences.to_csv('wolof_translate/data/sentences/wolof_french.csv', index=False)
         
-        modified.to_csv('wolof-translate/wolof_translate/data/sentences/deleted_lines.csv', index=False)
+        modified.to_csv('wolof_translate/data/sentences/deleted_lines.csv', index=False)
 
 
 left.header("French")

pages/provide_sentences_mongo.py

@@ -20,15 +20,15 @@ sentences_, deleted = db_manager.load_data_frames()
 sentences = sentences_.copy()
 
 # get french and wolof sentences
-french_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/french.csv')
+french_examples = pd.read_csv('wolof_translate/data/sentences/french.csv')
 
-wolof_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_2.csv')
+wolof_examples = pd.read_csv('wolof_translate/data/sentences/wolof_2.csv')
 
 # add special characters from Wolof
-sp_wolof_chars = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/wolof_special_chars.csv')
+sp_wolof_chars = pd.read_csv('wolof_translate/data/wolof_writing/wolof_special_chars.csv')
 
 # add definitions
-sp_wolof_words = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/definitions.csv')
+sp_wolof_words = pd.read_csv('wolof_translate/data/wolof_writing/definitions.csv')
 
 sp_wolof_words.sort_values(by = ['french', 'wolof'], inplace = True)
 
@@ -93,10 +93,10 @@ def add_new_sentences():
         
         
         ## save the result
-        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        sentences.to_csv('wolof_translate/data/sentences/wolof_french.csv', index=False)
         
         # recuperate the already saved sentences (for the moment french/wolof)
-        sentences = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')
+        sentences = pd.read_csv('wolof_translate/data/sentences/wolof_french.csv')
         
         # add the last position to delete and modify
         st.session_state.line1 = len(sentences) - 1

requirements.txt

@@ -1,5 +1,4 @@
 altair<5
 transformers
 torch
-pandas
-wolof-translate
+pandas

send_and_retrieve_data.ipynb

@@ -670,7 +670,6 @@
     }
    ],
    "source": [
-    "# %%writefile wolof-translate/wolof_translate/utils/database_manager.py\n",
     "from pymongo.mongo_client import MongoClient\n",
     "from pymongo.server_api import ServerApi\n",
     "import pandas as pd\n",

wolof-translate/setup.py

@@ -1,5 +0,0 @@
-from setuptools import setup
-
-setup(name="wolof_translate", version="0.0.1", author="Oumar Kane", author_email="oumar.kane@univ-thies.sn", 
-      description="Contain function and classes to process corpora for making translation between wolof text and other languages.",
-      requires=['spacy', 'nltk', 'gensim'])

wolof-translate/wolof_translate.egg-info/PKG-INFO

@@ -1,9 +0,0 @@
-Metadata-Version: 2.1
-Name: wolof-translate
-Version: 0.0.1
-Summary: Contain function and classes to process corpora for making translation between wolof text and other languages.
-Author: Oumar Kane
-Author-email: oumar.kane@univ-thies.sn
-Requires: spacy
-Requires: nltk
-Requires: gensim

wolof-translate/wolof_translate.egg-info/SOURCES.txt

@@ -1,59 +0,0 @@
-setup.py
-wolof_translate/__init__.py
-wolof_translate.egg-info/PKG-INFO
-wolof_translate.egg-info/SOURCES.txt
-wolof_translate.egg-info/dependency_links.txt
-wolof_translate.egg-info/top_level.txt
-wolof_translate/__pycache__/__init__.cpython-310.pyc
-wolof_translate/__pycache__/dataset_v1.cpython-310.pyc
-wolof_translate/__pycache__/sent_transformers.cpython-310.pyc
-wolof_translate/data/__init__.py
-wolof_translate/data/dataset_v1.py
-wolof_translate/data/dataset_v2.py
-wolof_translate/data/dataset_v3.py
-wolof_translate/data/__pycache__/__init__.cpython-310.pyc
-wolof_translate/data/__pycache__/dataset_v1.cpython-310.pyc
-wolof_translate/data/__pycache__/dataset_v2.cpython-310.pyc
-wolof_translate/models/__init__.py
-wolof_translate/models/__pycache__/__init__.cpython-310.pyc
-wolof_translate/models/transformers/__init__.py
-wolof_translate/models/transformers/main.py
-wolof_translate/models/transformers/optimization.py
-wolof_translate/models/transformers/position.py
-wolof_translate/models/transformers/size.py
-wolof_translate/models/transformers/__pycache__/__init__.cpython-310.pyc
-wolof_translate/models/transformers/__pycache__/main.cpython-310.pyc
-wolof_translate/models/transformers/__pycache__/optimization.cpython-310.pyc
-wolof_translate/models/transformers/__pycache__/position.cpython-310.pyc
-wolof_translate/models/transformers/__pycache__/size.cpython-310.pyc
-wolof_translate/pipe/__init__.py
-wolof_translate/pipe/nlp_pipeline.py
-wolof_translate/tokenizers/__init__.py
-wolof_translate/tokenizers/adverse_tokenizer.json
-wolof_translate/tokenizers/tokenizer_v1.json
-wolof_translate/tokenizers/__pycache__/__init__.cpython-310.pyc
-wolof_translate/tokenizers/t5_tokenizers/fr_tokenizer_v1.json
-wolof_translate/tokenizers/t5_tokenizers/tokenizer_v1.json
-wolof_translate/tokenizers/t5_tokenizers/tokenizer_v2.json
-wolof_translate/tokenizers/t5_tokenizers/tokenizer_v3.json
-wolof_translate/tokenizers/t5_tokenizers/wf_tokenizer_v1.json
-wolof_translate/trainers/__init__.py
-wolof_translate/trainers/transformer_trainer.py
-wolof_translate/trainers/__pycache__/__init__.cpython-310.pyc
-wolof_translate/trainers/__pycache__/transformer_trainer.cpython-310.pyc
-wolof_translate/utils/__init__.py
-wolof_translate/utils/evaluation.py
-wolof_translate/utils/extract_poems.py
-wolof_translate/utils/extract_sentences.py
-wolof_translate/utils/sent_corrections.py
-wolof_translate/utils/sent_transformers.py
-wolof_translate/utils/sent_unification.py
-wolof_translate/utils/split_with_valid.py
-wolof_translate/utils/tokenize_text.py
-wolof_translate/utils/__pycache__/__init__.cpython-310.pyc
-wolof_translate/utils/__pycache__/evaluation.cpython-310.pyc
-wolof_translate/utils/__pycache__/sent_corrections.cpython-310.pyc
-wolof_translate/utils/__pycache__/sent_transformers.cpython-310.pyc
-wolof_translate/utils/__pycache__/sent_unification.cpython-310.pyc
-wolof_translate/utils/__pycache__/split_with_valid.cpython-310.pyc
-wolof_translate/utils/__pycache__/tokenize_text.cpython-310.pyc

wolof-translate/wolof_translate.egg-info/dependency_links.txt

@@ -1 +0,0 @@
-

wolof-translate/wolof_translate.egg-info/top_level.txt

@@ -1 +0,0 @@
-wolof_translate

wolof-translate/wolof_translate/checkpoints/t5_base_custom_train_results_fw_v3/best_checkpoints.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3c27c98fb26335a018a9e39fa61efe4b16e356439d266fee60f41e7c30ac9e95
-size 801021373

wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3/best_checkpoints.json

@@ -1,13 +0,0 @@
-{
-    "metrics": {
-        "train_loss": 0.003311718240206019,
-        "test_loss": 0.5539475262048654,
-        "bleu": 28.528,
-        "gen_len": 8.1575,
-        "current_epoch": 346
-    },
-    "best_performance": {
-        "best_score": 28.528,
-        "best_epoch": 346
-    }
-}

wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_fw_v3/best_checkpoints.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b30a787b8488fb4f230d37ee9fab553e95e18fbf7e7036920069997a5b975534
-size 180980359

wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3/best_checkpoints.json

@@ -1,13 +0,0 @@
-{
-    "metrics": {
-        "train_loss": 0.010352298853033019,
-        "test_loss": 0.5878886595368386,
-        "bleu": 22.9625,
-        "gen_len": 9.3562,
-        "current_epoch": 94
-    },
-    "best_performance": {
-        "best_score": 22.9625,
-        "best_epoch": 94
-    }
-}

wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3/best_checkpoints.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5e543a1882f682a4fa84f6c6ff4d21e8afceb0cbee401bd934c7338f7515320f
-size 180980103