test database in huggingface

.gitignore

@@ -0,0 +1 @@
+pages/provide_sentences.py

app.py

@@ -74,7 +74,7 @@ st.markdown("Translation page 🔠")
 #     max_len = models[version][translation_type]['max_len']
     
 #     # let us get the best model
-#     # @st.cache_resource
+    # @st.cache_resource
 #     def get_modelfw_v3():
         
 #         # recuperate checkpoints

pages/provide_sentences.py

@@ -11,7 +11,7 @@ sentences = sentences_.copy()
 # get french and wolof sentences
 french_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/french.csv')
 
-wolof_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof.csv')
+wolof_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_2.csv')
 
 # add special characters from Wolof
 sp_wolof_chars = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/wolof_special_chars.csv')

pages/provide_sentences_mongo.py

@@ -0,0 +1,246 @@
+from wolof_translate.utils.database_manager import TranslationMongoDBManager
+import streamlit as st
+import pandas as pd
+
+st.markdown("Provide your own 🤗 sentences")
+
+# let us initialize the database manage
+@st.cache_resource
+def get_cluster():
+    
+    db_manager = TranslationMongoDBManager('mongodb+srv://oumar199:Jacksparrow360@woloftranslationcluster.u0gk7.mongodb.net/?retryWrites=true&w=majority', 'WolofTranslation')
+
+    return db_manager
+
+db_manager = get_cluster()
+
+# recuperate the already saved sentences (for the moment french/wolof sentences)
+sentences_, deleted = db_manager.load_data_frames()
+
+sentences = sentences_.copy()
+
+# get french and wolof sentences
+french_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/french.csv')
+
+wolof_examples = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_2.csv')
+
+# add special characters from Wolof
+sp_wolof_chars = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/wolof_special_chars.csv')
+
+# add definitions
+sp_wolof_words = pd.read_csv('wolof-translate/wolof_translate/data/wolof_writing/definitions.csv')
+
+sp_wolof_words.sort_values(by = ['french', 'wolof'], inplace = True)
+
+# initialize the input texts
+st.title("Provide sentences below ⤵️")
+
+st.markdown("""---""")
+
+# create three columns
+left, right = st.columns(2)
+
+# let us add a callback functions to change the input text
+def add_symbol_to_french():
+    
+    st.session_state.left_sentence += st.session_state.fr_symbol
+    
+def add_symbol_to_wolof():
+    
+    st.session_state.right_sentence += st.session_state.wf_symbol
+
+def add_special_token_french():
+    
+    st.session_state.left_sentence += '<mask>'
+    
+def add_special_token_wolof():
+    
+    st.session_state.right_sentence += '<mask>'
+
+def add_word_to_text():
+    
+    word = st.session_state.word.split('/')[0].strip()
+    
+    st.session_state.right_sentence += word
+
+def add_french_sentence_to_text():
+    
+    sentence = st.session_state.french_sentence.strip()
+    
+    st.session_state.left_sentence = sentence
+    
+def add_wolof_sentence_to_text():
+    
+    sentence = st.session_state.wolof_sentence.strip()
+    
+    st.session_state.right_sentence = sentence
+
+# let us create a callback which permit us to add sentences inside a DataFrame
+def add_new_sentences():
+    
+    global sentences
+    
+    sentence_1 = st.session_state.left_sentence.strip()
+    
+    sentence_2 = st.session_state.right_sentence.strip()
+    
+    if sentence_1 == '' or sentence_2 == '':
+        
+        st.warning("You didn't provide a sentence ! Please provide before submitting.", icon= "🚨")
+    
+    else:
+        
+        
+        
+        ## save the result
+        sentences.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)
+        
+        # recuperate the already saved sentences (for the moment french/wolof)
+        sentences = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')
+        
+        # add the last position to delete and modify
+        st.session_state.line1 = len(sentences) - 1
+        
+        st.session_state.line2 = len(sentences) - 1
+        
+# let us create a callback which permit us to add sentences inside a DataFrame
+def add_new_sentences2():
+    
+    global sentences
+    
+    sentence_1 = st.session_state.left_sentence.strip()
+    
+    sentence_2 = st.session_state.right_sentence.strip()
+    
+    if sentence_1 == '' or sentence_2 == '':
+        
+        st.warning("You didn't provide a sentence ! Please provide before submitting.", icon= "🚨")
+    
+    else:
+        
+        # insert the new sentences
+        db_manager.insert_document({
+            'french': sentence_1,
+            'wolof': sentence_2
+        })
+        
+        # recuperate the already saved sentences (for the moment french/wolof)
+        sentences = db_manager.load_data_frames()
+        
+        # clean the inputs' contents
+        st.session_state.left_sentence = ''
+        st.session_state.right_sentence = ''
+        
+        # add the last position to delete and modify
+        st.session_state.line1 = len(sentences) - 1
+        
+        st.session_state.line2 = len(sentences) - 1
+        
+def delete_line():
+    
+    global sentences
+    
+    number = st.session_state.line1
+    
+    if number > len(sentences) - 1:
+        
+        st.warning("The line that you provided does not exist !")
+    
+    else:
+        
+        # delete a document
+        db_manager.delete_document(number)
+        
+        # recuperate the already saved sentences (for the moment french/wolof)
+        sentences = db_manager.load_data_frames()
+
+def modify_line():
+    
+    global sentences
+    
+    number = st.session_state.line2
+    
+    if number > len(sentences) - 1:
+        
+        st.warning("The line that you provided does not exist !")
+    
+    else:
+        st.session_state.left_sentence = sentences.loc[number, sentences.columns.tolist()[1]]
+        
+        st.session_state.right_sentence = sentences.loc[number, sentences.columns.tolist()[2]]
+
+def update_sentences():
+    
+    global sentences
+    
+    number = st.session_state.line2
+    
+    sentence_1 = st.session_state.left_sentence.strip()
+    
+    sentence_2 = st.session_state.right_sentence.strip()
+    
+    if number > len(sentences) - 1:
+        
+        st.warning("The line that you provided does not exist !")
+    
+    elif sentence_1 == '' or sentence_2 == '':
+        
+        st.warning("You didn't provide a sentence ! Please provide before submitting.", icon= "🚨")
+    
+    else:
+        
+        db_manager.update_document(number, 
+                                   {
+                                       'french': sentence_1,
+                                       'wolof': sentence_2
+                                   })
+        
+        # recuperate the already saved sentences (for the moment french/wolof)
+        sentences = db_manager.load_data_frames()
+
+left.header("French")
+left.text_area(sentences.columns.tolist()[1], key = "left_sentence")
+
+right.header("Wolof")
+right.text_area(sentences.columns.tolist()[2], key = "right_sentence")
+
+fr_symbol = left.selectbox("French characters", key="fr_symbol", options = sp_wolof_chars['wolof_special_chars'])
+
+wf_symbol = right.selectbox("Wolof characters", key="wf_symbol", options = sp_wolof_chars['wolof_special_chars'])
+
+word = right.selectbox("Wolof words/Definitions", key="word", options = [sp_wolof_words.loc[i, 'wolof']+" / "+sp_wolof_words.loc[i, 'french'] for i in range(sp_wolof_words.shape[0])], on_change=add_word_to_text)
+
+left.button("Add symbol", "add fr symbol", on_click=add_symbol_to_french)
+
+left.button("Add mask", "french mask", on_click=add_special_token_french)
+
+right.button("Add symbol", "add wf symbol", on_click=add_symbol_to_wolof)
+
+right.button("Add mask", "wolof mask", on_click=add_special_token_wolof)
+
+# add sentences at sidebar
+st.sidebar.selectbox("French sentences", key="french_sentence", options = french_examples['sentences'], on_change=add_french_sentence_to_text)
+
+st.sidebar.selectbox("Wolof sentences", key="wolof_sentence", options = wolof_examples['sentences'], on_change=add_wolof_sentence_to_text)
+
+# add a submit button to add new sentences
+st.button('Submit', 'submit_button', on_click=add_new_sentences)
+st.button('Place', 'submit_button2', on_click=add_new_sentences2)
+st.button('Update', 'submit_button3', on_click=update_sentences)
+
+st.markdown("""---""")
+
+# add delete button
+number = st.number_input("Provide line to delete", key="line1", min_value=0)
+
+st.button("Delete", 'delete_button', on_click=delete_line)
+
+# add modify button
+number = st.number_input("Provide line to modify", key="line2", min_value=0)
+
+st.button("Modify", 'modify_button', on_click=modify_line)
+
+# add data frame
+st.dataframe(sentences.set_index('_id'), width=900)
+
+    

recuperate_huggingface_data.ipynb

@@ -19,7 +19,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -83,6 +83,14 @@
     "- Wolof sentences"
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "1. First one hundred"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -149,7 +157,83 @@
    "outputs": [],
    "source": [
     "# save the sentences\n",
-    "pd.DataFrame({'sentences': wolof_sents}).to_csv('wolof.csv', index=False)"
+    "pd.DataFrame({'sentences': wolof_sents}).to_csv('wolof_1.csv', index=False)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "2. second ..."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "curl = \"https://datasets-server.huggingface.co/rows?dataset=perrynelson%2Fwaxal-wolof&config=perrynelson--waxal-wolof&split=test&offset=100&limit=100\"\n",
+    "\n",
+    "def query():\n",
+    "    \n",
+    "    response = requests.get(curl)\n",
+    "    return response.json()\n",
+    "\n",
+    "data = query()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Get the transcriptions in a data frame."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "100"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data['rows'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sentences = data['rows']\n",
+    "\n",
+    "wolof_sents = []\n",
+    "\n",
+    "for i in range(len(sentences)):\n",
+    "    \n",
+    "    wolof_sents.append(sentences[i]['row']['transcription'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save the sentences\n",
+    "pd.DataFrame({'sentences': wolof_sents}).to_csv('wolof_2.csv', index=False)"
    ]
   }
  ],

requirements.txt

@@ -1,4 +1,5 @@
 altair<5
 transformers
 torch
-pandas
+pandas
+wolof-translate

send_and_retrieve_data.ipynb

@@ -0,0 +1,828 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "MongoDB Cluster management\n",
+    "-----------------------\n",
+    "\n",
+    "Let us create some function to send (insert documents) and retrieve collections from our cluster database."
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We must import the pymongo the client and the server modules."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 101,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pymongo.mongo_client import MongoClient\n",
+    "from pymongo.server_api import ServerApi\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# let us set below the cluster uri\n",
+    "uri = \"mongodb+srv://oumar199:Jacksparrow360@woloftranslationcluster.u0gk7.mongodb.net/?retryWrites=true&w=majority\""
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Inserting new documents"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let us insert new documents to our collection."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 103,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# we must initialize the client\n",
+    "client = MongoClient(uri, server_api = ServerApi('1'))"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let us create a new collection."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create a database\n",
+    "db = client.get_database('WolofTranslation')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 105,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create a new collection for the new sentences\n",
+    "sentences = db.sentences"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is time to insert the new documents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 106,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# recuperate the already created sentences\n",
+    "corpora = pd.read_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 107,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# let us reset the indices\n",
+    "corpora.reset_index(inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 108,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>french</th>\n",
+       "      <th>wolof</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>J'arrive tout de suite chez toi.</td>\n",
+       "      <td>Léegui léegui ma egg sa kër.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>J'en suis sûr, cette photo ci c'est la photo p...</td>\n",
+       "      <td>Waaw nataal bii nataal la boob ay nit ñu baree...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Je vois devant moi une photo sur laquelle beau...</td>\n",
+       "      <td>Nataal bii maa ngi ciy janloog haa ay nit yu b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Ceux-ci sont des personnes qui sont sortis pou...</td>\n",
+       "      <td>Lii, ay nit lañu yu génn di ñaxtu. Jëm yi nag ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Salut ! Ceux-là qui ressemblent à des personne...</td>\n",
+       "      <td>Salaawaalekum ! Ñii de, mel nañ ne, ay nit ñu ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>5</td>\n",
+       "      <td>Cette photo ci c'est une photo sur laquelle je...</td>\n",
+       "      <td>Nataal bi ab nataal la boo xamante yni maa ngi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>6</td>\n",
+       "      <td>Sur la photo, ont voit des personnes qui se ré...</td>\n",
+       "      <td>Nataal bii ñoo ngi ciy gis ay nit ñuy ñaxtu wa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>7</td>\n",
+       "      <td>On voit sur la photo beaucoup de personnes sor...</td>\n",
+       "      <td>Ñu gis ci nataal bi ay nit ñu bari ñu génn ci ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>8</td>\n",
+       "      <td>C'est des poissons, oui. Ils sont de couleur b...</td>\n",
+       "      <td>Jën la waaw, Wu am wirgo Wu baxa ak Wu xonq.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>9</td>\n",
+       "      <td>Ah sur cette photo ci cependant, il y a un poi...</td>\n",
+       "      <td>Aah nataal bii nag, aw jën la. Jën wi mi ngi a...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   index                                             french  \\\n",
+       "0      0                   J'arrive tout de suite chez toi.   \n",
+       "1      1  J'en suis sûr, cette photo ci c'est la photo p...   \n",
+       "2      2  Je vois devant moi une photo sur laquelle beau...   \n",
+       "3      3  Ceux-ci sont des personnes qui sont sortis pou...   \n",
+       "4      4  Salut ! Ceux-là qui ressemblent à des personne...   \n",
+       "5      5  Cette photo ci c'est une photo sur laquelle je...   \n",
+       "6      6  Sur la photo, ont voit des personnes qui se ré...   \n",
+       "7      7  On voit sur la photo beaucoup de personnes sor...   \n",
+       "8      8  C'est des poissons, oui. Ils sont de couleur b...   \n",
+       "9      9  Ah sur cette photo ci cependant, il y a un poi...   \n",
+       "\n",
+       "                                               wolof  \n",
+       "0                       Léegui léegui ma egg sa kër.  \n",
+       "1  Waaw nataal bii nataal la boob ay nit ñu baree...  \n",
+       "2  Nataal bii maa ngi ciy janloog haa ay nit yu b...  \n",
+       "3  Lii, ay nit lañu yu génn di ñaxtu. Jëm yi nag ...  \n",
+       "4  Salaawaalekum ! Ñii de, mel nañ ne, ay nit ñu ...  \n",
+       "5  Nataal bi ab nataal la boo xamante yni maa ngi...  \n",
+       "6  Nataal bii ñoo ngi ciy gis ay nit ñuy ñaxtu wa...  \n",
+       "7  Ñu gis ci nataal bi ay nit ñu bari ñu génn ci ...  \n",
+       "8       Jën la waaw, Wu am wirgo Wu baxa ak Wu xonq.  \n",
+       "9  Aah nataal bii nag, aw jën la. Jën wi mi ngi a...  "
+      ]
+     },
+     "execution_count": 108,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "corpora.head(10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 109,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# insert the sentences\n",
+    "results = sentences.insert_many({\n",
+    "        '_id': corp, # set the id\n",
+    "        'french': corpora.loc[corp, 'french'],\n",
+    "        'wolof': corpora.loc[corp, 'wolof']\n",
+    "    } for corp in corpora.index\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let us insert the deleted sentences."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 110,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# create a new collection named deleted.\n",
+    "deleted = db.deleted"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 111,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# recuperated the data frame of deleted sentences\n",
+    "del_corpora = pd.read_csv('wolof-translate/wolof_translate/data/sentences/deleted_lines.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 112,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# reset the indices\n",
+    "del_corpora.reset_index(inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 113,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# insert the deleted sentences\n",
+    "results = deleted.insert_many({\n",
+    "        '_id': corp, # set the id\n",
+    "        'french': corpora.loc[corp, 'french'],\n",
+    "        'wolof': corpora.loc[corp, 'wolof']\n",
+    "    } for corp in del_corpora.index\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Modify sentences"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We want to modify only one sentence at a time."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 114,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# select the id to modify\n",
+    "id_ = 1\n",
+    "\n",
+    "# retrieve new sentences\n",
+    "french = corpora.loc[id_, 'french']\n",
+    "wolof = corpora.loc[id_, 'wolof']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 115,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(\"J'en suis sûr, cette photo ci c'est la photo pris au moment où plusieurs personnes font une marche de révolte tendant leurs mains. Ceux là sont assis, ceux là sont debout entrain de marcher. On a écrit sur la photo quelque chose de bleu concernant la Casamance.\",\n",
+       " 'Waaw nataal bii nataal la boob ay nit ñu baree bari ñoo xam ni dañuy doxub ñaxtu ñoo ci nekk tàllal seen i loxo. Ñee sukku ñeel taxaw jodd di dox. Ñu bind ci kaw nataal bi lu xaw a baxa la bind ci laa kaasamãs.')"
+      ]
+     },
+     "execution_count": 115,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# print the sentences\n",
+    "french, wolof"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 118,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# modify the sentences at the id\n",
+    "results = sentences.update_one(\n",
+    "    {\n",
+    "        '_id': {'$eq': id_}\n",
+    "    },\n",
+    "    {\n",
+    "        '$set': {\n",
+    "            'french': french,\n",
+    "            'wolof': wolof + \"--------\" # we added a modification\n",
+    "        }\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let us show the first documents."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 119,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>_id</th>\n",
+       "      <th>french</th>\n",
+       "      <th>wolof</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>J'arrive tout de suite chez toi.</td>\n",
+       "      <td>Léegui léegui ma egg sa kër.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>J'en suis sûr, cette photo ci c'est la photo p...</td>\n",
+       "      <td>Waaw nataal bii nataal la boob ay nit ñu baree...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>Je vois devant moi une photo sur laquelle beau...</td>\n",
+       "      <td>Nataal bii maa ngi ciy janloog haa ay nit yu b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>Ceux-ci sont des personnes qui sont sortis pou...</td>\n",
+       "      <td>Lii, ay nit lañu yu génn di ñaxtu. Jëm yi nag ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>Salut ! Ceux-là qui ressemblent à des personne...</td>\n",
+       "      <td>Salaawaalekum ! Ñii de, mel nañ ne, ay nit ñu ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>5</td>\n",
+       "      <td>Cette photo ci c'est une photo sur laquelle je...</td>\n",
+       "      <td>Nataal bi ab nataal la boo xamante yni maa ngi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>6</td>\n",
+       "      <td>Sur la photo, ont voit des personnes qui se ré...</td>\n",
+       "      <td>Nataal bii ñoo ngi ciy gis ay nit ñuy ñaxtu wa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>7</td>\n",
+       "      <td>On voit sur la photo beaucoup de personnes sor...</td>\n",
+       "      <td>Ñu gis ci nataal bi ay nit ñu bari ñu génn ci ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>8</td>\n",
+       "      <td>C'est des poissons, oui. Ils sont de couleur b...</td>\n",
+       "      <td>Jën la waaw, Wu am wirgo Wu baxa ak Wu xonq.</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>9</td>\n",
+       "      <td>Ah sur cette photo ci cependant, il y a un poi...</td>\n",
+       "      <td>Aah nataal bii nag, aw jën la. Jën wi mi ngi a...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   _id                                             french  \\\n",
+       "0    0                   J'arrive tout de suite chez toi.   \n",
+       "1    1  J'en suis sûr, cette photo ci c'est la photo p...   \n",
+       "2    2  Je vois devant moi une photo sur laquelle beau...   \n",
+       "3    3  Ceux-ci sont des personnes qui sont sortis pou...   \n",
+       "4    4  Salut ! Ceux-là qui ressemblent à des personne...   \n",
+       "5    5  Cette photo ci c'est une photo sur laquelle je...   \n",
+       "6    6  Sur la photo, ont voit des personnes qui se ré...   \n",
+       "7    7  On voit sur la photo beaucoup de personnes sor...   \n",
+       "8    8  C'est des poissons, oui. Ils sont de couleur b...   \n",
+       "9    9  Ah sur cette photo ci cependant, il y a un poi...   \n",
+       "\n",
+       "                                               wolof  \n",
+       "0                       Léegui léegui ma egg sa kër.  \n",
+       "1  Waaw nataal bii nataal la boob ay nit ñu baree...  \n",
+       "2  Nataal bii maa ngi ciy janloog haa ay nit yu b...  \n",
+       "3  Lii, ay nit lañu yu génn di ñaxtu. Jëm yi nag ...  \n",
+       "4  Salaawaalekum ! Ñii de, mel nañ ne, ay nit ñu ...  \n",
+       "5  Nataal bi ab nataal la boo xamante yni maa ngi...  \n",
+       "6  Nataal bii ñoo ngi ciy gis ay nit ñuy ñaxtu wa...  \n",
+       "7  Ñu gis ci nataal bi ay nit ñu bari ñu génn ci ...  \n",
+       "8       Jën la waaw, Wu am wirgo Wu baxa ak Wu xonq.  \n",
+       "9  Aah nataal bii nag, aw jën la. Jën wi mi ngi a...  "
+      ]
+     },
+     "execution_count": 119,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get the 10 first sentences into a Data Frame\n",
+    "pd.DataFrame(list(sentences.find().limit(10)))"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Delete sentences"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We want to modify only one sentence at a time. The deleted sentences must be added into the 'deleted' collection. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# recuperate the sentences to delete (id = 0)\n",
+    "id_ = 0\n",
+    "\n",
+    "del_sent = sentences.find_one(\n",
+    "    {\n",
+    "        '_id': {'$eq': id_}\n",
+    "    }    \n",
+    ")\n",
+    "\n",
+    "# delete the sentence and add it into the deleted sentences\n",
+    "sentences.delete_one(\n",
+    "    {\n",
+    "        '_id': {'$eq': del_sent['_id']}\n",
+    "    }\n",
+    ")\n",
+    "\n",
+    "results = deleted.insert_one(\n",
+    "    {\n",
+    "        '_id': len(list(deleted.find())),\n",
+    "        'french': del_sent['french'],\n",
+    "        'wolof': del_sent['wolof']\n",
+    "    }\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### From collection to DataFrame"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We must convert the sentences to csv files in order to use at the training step."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 123,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# recuperate the new corpora\n",
+    "new_corpora = pd.DataFrame(list(sentences.find()))\n",
+    "\n",
+    "# recuperate the deleted sentences as a Data Frame\n",
+    "deleted_df = pd.DataFrame(list(deleted.find()))\n",
+    "\n",
+    "# save the data frames as csv files\n",
+    "new_corpora.set_index('_id', inplace=True)\n",
+    "\n",
+    "deleted_df.set_index('_id', inplace=True)\n",
+    "\n",
+    "new_corpora.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index=False)\n",
+    "\n",
+    "deleted_df.to_csv('wolof-translate/wolof_translate/data/sentences/deleted_lines.csv', index=False)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### All in one"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let us create a class which consider each of the methods we investigated previously."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 129,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'_id': 150,\n",
+       " 'french': \"Sur la photo que vous m'avez envoyée, j'ai vu qu'il s'agissait de gendarmes. Des gendarmes qui portent, cependant, des... des boucliers. Des verres conçus pour les protéger. Ils sont faces au peuple s'échangeant contre eux des cailloux et des pierres.\",\n",
+       " 'wolof': 'Nataal bi ngeen ma yonnee, gis naa ni ay takk-der la. Takk der yoo xamantane bii nag, jël nañ loo xamantane bii mooy ay,... Ay baar. Ay verre yoo xam ne  dañ kaa defar pour ñu leen di baare, ñu jàkkarlook askan wi di sànnanteek ñoom ay xeer ak ay doj.'}"
+      ]
+     },
+     "execution_count": 129,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "list(sentences.find().sort('_id', -1).limit(1))[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 132,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Overwriting wolof-translate/wolof_translate/utils/database_manager.py\n"
+     ]
+    }
+   ],
+   "source": [
+    "# %%writefile wolof-translate/wolof_translate/utils/database_manager.py\n",
+    "from pymongo.mongo_client import MongoClient\n",
+    "from pymongo.server_api import ServerApi\n",
+    "import pandas as pd\n",
+    "\n",
+    "class TranslationMongoDBManager:\n",
+    "    \n",
+    "    def __init__(self, uri: str, database: str):\n",
+    "        \n",
+    "        # recuperate the client\n",
+    "        self.client = MongoClient(uri)\n",
+    "        \n",
+    "        # recuperate the database\n",
+    "        self.db = self.client.get_database(database)\n",
+    "        \n",
+    "    def insert_documents(self, documents: list, collection: str = \"sentences\"):\n",
+    "        \n",
+    "        # insert documents inside a collection\n",
+    "        results = self.db[collection].insert_many(documents)\n",
+    "        \n",
+    "        return results\n",
+    "    \n",
+    "    def insert_document(self, document: dict, collection: str = \"sentences\"):\n",
+    "        \n",
+    "        assert '_id' in document\n",
+    "        \n",
+    "        # get the id of the last sentence (recuperate the max id and add 1 to it)\n",
+    "        max_id = self.get_max_id(collection)\n",
+    "        \n",
+    "        # add the new sentences\n",
+    "        document['_id'] = max_id\n",
+    "        \n",
+    "        results = self.db[collection].insert_one(\n",
+    "            document\n",
+    "        )\n",
+    "        \n",
+    "        return results\n",
+    "    \n",
+    "    def update_document(self, id: int, document: dict, collection: str = \"sentences\", update_collection: str = \"updated\"):\n",
+    "        \n",
+    "        # recuperate the document to update\n",
+    "        upd_sent = self.db[collection].find_one(\n",
+    "            {\n",
+    "                '_id': {\n",
+    "                    '$eq': id\n",
+    "                }\n",
+    "            }\n",
+    "        )\n",
+    "        \n",
+    "        # update the document\n",
+    "        results = self.db[collection].update_one(\n",
+    "            {\n",
+    "                '_id': {\n",
+    "                    '$eq': id\n",
+    "                }\n",
+    "            },\n",
+    "            {\n",
+    "                '$set': document\n",
+    "            }\n",
+    "        )\n",
+    "        \n",
+    "        # add the sentences to the deleted sentences\n",
+    "        upd_sent['_id'] = len(list(self.db[update_collection].find()))\n",
+    "        \n",
+    "        results = self.db[update_collection].insert_one(\n",
+    "            upd_sent\n",
+    "        )\n",
+    "        \n",
+    "        return results\n",
+    "    \n",
+    "    def delete_document(self, id: int, collection: str = \"sentences\", del_collection: str = \"deleted\"):\n",
+    "        \n",
+    "        # recuperate the document to delete\n",
+    "        del_sent = self.db[collection].find_one(\n",
+    "            {\n",
+    "                '_id': {\n",
+    "                    '$eq': id\n",
+    "                }\n",
+    "            }\n",
+    "        )\n",
+    "        \n",
+    "        # delete the sentence\n",
+    "        self.db[collection].delete_one(\n",
+    "            {\n",
+    "                '_id': {'$eq': del_sent['_id']}\n",
+    "            }\n",
+    "        )\n",
+    "        \n",
+    "        # add the sentences to the deleted sentences\n",
+    "        del_sent['_id'] = len(list(self.db[del_collection].find()))\n",
+    "        \n",
+    "        results = self.db[del_collection].insert_one(\n",
+    "            del_sent\n",
+    "        )\n",
+    "        \n",
+    "        return results\n",
+    "    \n",
+    "    def get_max_id(self, collection):\n",
+    "        \n",
+    "        # recuperate the maximum id\n",
+    "        id = list(self.db[collection].find().sort('_id', -1).limit(1))[0]['_id']\n",
+    "        \n",
+    "        return id\n",
+    "    \n",
+    "    def save_data_frames(self, sentences_path: str, deleted_path: str, collection: str = \"sentences\", del_collection: str = \"deleted\"):\n",
+    "        \n",
+    "        # recuperate the new corpora\n",
+    "        new_corpora = pd.DataFrame(list(self.db[collection].find()))\n",
+    "\n",
+    "        # recuperate the deleted sentences as a Data Frame\n",
+    "        deleted_df = pd.DataFrame(list(self.db[del_collection].find()))\n",
+    "\n",
+    "        # save the data frames as csv files\n",
+    "        new_corpora.set_index('_id', inplace=True)\n",
+    "\n",
+    "        deleted_df.set_index('_id', inplace=True)\n",
+    "\n",
+    "        new_corpora.to_csv(sentences_path, index=False)\n",
+    "\n",
+    "        deleted_df.to_csv(deleted_path, index=False)\n",
+    "    \n",
+    "    def load_data_frames(self, collection: str = \"sentences\", del_collection: str = \"deleted\"):\n",
+    "        \n",
+    "        # recuperate the new corpora\n",
+    "        new_corpora = pd.DataFrame(list(self.db[collection].find()))\n",
+    "\n",
+    "        # recuperate the deleted sentences as a Data Frame\n",
+    "        deleted_df = pd.DataFrame(list(self.db[del_collection].find()))\n",
+    "        \n",
+    "        return new_corpora, deleted_df"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "pytorch1-HleOW5am-py3.10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

upper_case_new_sent.py

@@ -0,0 +1,29 @@
+import pandas as pd
+sentences = pd.read_csv("wolof-translate/wolof_translate/data/sentences/wolof_french.csv")
+french = sentences['french']
+wolof = sentences['wolof']
+new_french = []
+new_wolof = []
+
+for sent in french:
+     letters = [sent[0], sent[1]]
+     for l in sent[2:]:
+             if letters[-1] in ['?', '!', '.'] or letters[-2] in ['?', '.', '!'] and l.isupper():
+                     letters.append(l.upper())
+             else:
+                     letters.append(l)
+     new_french.append("".join(letters))
+ 
+for sent in wolof:                                                            
+     letters = [sent[0], sent[1]]
+     for l in sent[2:]:
+             if letters[-1] in ['?', '!', '.'] or letters[-2] in ['?', '.', '!'] and l.isupper():
+                     letters.append(l.upper())                                 
+             else:
+                     letters.append(l)
+     new_wolof.append("".join(letters))
+
+new_sents = pd.DataFrame({'french': new_french, 'wolof': new_wolof})
+new_sents.to_csv('wolof-translate/wolof_translate/data/sentences/wolof_french.csv', index = False)
+# print(new_sents.head(10))
+

wolof-translate/wolof_translate/checkpoints/t5_base_custom_train_results_fw_v3/best_checkpoints.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c27c98fb26335a018a9e39fa61efe4b16e356439d266fee60f41e7c30ac9e95
+size 801021373

wolof-translate/wolof_translate/checkpoints/t5_small_custom_train_results_wf_v3/best_checkpoints.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0427e418e75d5842f8b95ebe9025e91d6e16dd79ab7d6f5815320e239e8b350f
-size 180980359
+oid sha256:5e543a1882f682a4fa84f6c6ff4d21e8afceb0cbee401bd934c7338f7515320f
+size 180980103

wolof-translate/wolof_translate/data/sentences/deleted_lines.csv

@@ -1,2 +1,2 @@
 french,wolof
-J'ai vu quelque chose qu'on achète au marché ...,Gis naa benn affaire bu ñuy ndugge marché ...
+J'arrive tout de suite chez toi.,Léegui léegui ma egg sa kër.

wolof-translate/wolof_translate/data/sentences/wolof_2.csv

@@ -0,0 +1,101 @@
+sentences
+"nataal bii ngeen ma yónnee de, gis naa ci biir nataal bi ab buntu bu rafet ñu liggéeyee ko dénk ak weer. am na ca kaw bunt ba limiyeer buy leer"
+"Nataal bii ñu ma yónnee gis naa ci ab kaadar def ay dàll, dàll yoo xam ne ay dàlli góor la, dàlli fermé yi tamit mi ngi ci."
+"Lii ay bunt la yoo xam ne ""menuisier métallique"" yi ñoo koy defar. Bunt yi nag dañ leen defar ba wéer leen daldi koy ""peinture"" ñu ame wirgo wu dóomu-taal. Mu mel na nag bunt bi ñaari bunt la  nga xam ne kon day mel ne bunt saal lay doon."
+"Waaw nataal bii de ay bunt yu dóomu-taal moo ci nekk. Bunt yi ëe dafa dóoomu-taal waawaw! Am na ay lu mel ni ay fu ngelaw di jaar boo ko xëccee waaw! Mën nga, mën nga ko xëcc, bunt yi waaw. Du benn bunt nag."
+Gis naa fi ab nataal bo xam ne ab lal la. Boo xam ne daa sokolaa am ñaari tànk. Am na wéeruwaay tamit. Loolu laa fi gis.
+Nataal bii ngeen ma yónnee de gis naa ci biir nataal bi ay toogu yu am tànki weñ te toogu ya daa ñuul.
+"Sama nataal bii ñu ma yónnee dafa mel ni ab sées la, sées yi ñuy tooge. Waaye nag couleur bi moom  dafa bula couleur bu bula la."
+"Nataal bii maa ngi ciy janook ""chaise"" bu nu liggéeye niki lu xaw a baxa. Moom toogukaay bii. Mel na ne yit ëe toogukaay la bu nu ràbbale ba fatt. Moom laay janool ci nataal bii ma janool."
+"Man de Li ma gis mooy benn lakku doj la. Yaakaar naa aw xeer ñu wërële ko ay, ay kii ay ""ciment"" ak suuf ak ndox, waawaaw!"
+"Waaw nataal bi de, ay xeer la yoo xam ni dañ dajaloo. Xeer yi nag Daf am am ay pax-pax ci biir, am ay cat-cat. Dafa xaw a marõ,xaw a tor lu mel ne kulëeru jën."
+"Xale yuy fo la, yor seeni fowukaay. Xale yi dañuy fo, yor seeni fowukaay. Ay xale yuy fo la."
+"Nataal bii ngeen ma yónnee de gis naa ci biir nataal bi jën yu mag, yuñ dagg ay dagg teg ko ci fenn."
+"Lii nag nataal la, nataalu gaas bu mag. Gaas bi gaas bu toggu la bu orãs am pupu ruus ci kaw ñu bind ci bind bu weex Total."
+"Nataal bii, maa ngi ciy janook ab gaas. Gaas boo xamantane bii ñoo ngi bind ci biir gaas bi Totaal. Gis naa ni gaas bu bees la bu nga xamantane bii masuñu koo jëfandikoo ndax benn kubéer bu ñuul bu nekk ci kaw."
+"Waa nataal bii nag moom ab furno laa siy gis ñu teg cin ci kaw, ak ab baŋ ci wet gi, ak ak nit ku taxaw, ak ñaari nit ñu taxaw, ak ay baŋ."
+"Waaw nataal bii de ngeen ma yónnee de nuru na lool furne ban, furne ban. Furne boo xam ne dañ koo defar ci ban, waaw moom laa gis ci nataal bi."
+Nataal bii ngeen ma yónnee gis naa ci biir nataal bi àndub ban bu am ay bën-bën ñu tabaxe ko ban.
+"Waaw ñi ngi siyaare bu baax, nataal bi maa ngi ci gis ay ay këll ak layu, ay këll ak layu yu rafet ak ñoom seen."
+"Li nga ma yónnee ba tey, li ma ci gis ay jëfandikukaay la, baraada la ak mësin pour tàngalukaay ak xellikaay la."
+Nataal bii ñaari tëddukaay la ñaari lal la. Benn ba teg nañ ca njegenaay lal ca darab. Benn bi ci suuf teguñu ci njegenaay kenn laluñ ci dara.
+"Waaw lii lañu naan beerso. Dénk lanu ko defaree, dénk.  Benn beerso bi nag ci biir lu mel ne xeetu darab bu weex moo ci nekk am benn njegeñaay. Beneen bi nag danu cee lal lu mel ne lu sokolaa ; darab bu sokolaa."
+"Nataal bii de nataal la boo xam ne verre la, verre bi nag mi ngi nekk weer weer yoo xam ne dañoo tegalante, verre yi nag benn bokku ceek moroomam waaw.Verre bi am na affaire bu ko tée boo xam ne mi ngi ci suuf, waaw affaire bu ko tée."
+"Waaw lii nii ab nataal bob mel na ni cemptang la boo xam ne dees koo defar, def ci ay weer. Cemptang gi itam mel na ni gis naa ci lu mel ni ki wañ... Woo xam ni dañ koo peentiir peentiir gu weex, dafal ko lu mel ni ki ak toog goo xam ni moo koy téye. Muy ñaari cemptang."
+"Nataalu juróom benn fukk ak juróom ñeent day wone mburu mu tëdd mel ne mburu mi lakk bi ñor na waaye ñorul lool. Ndax am na wuute genn wàll gi dafa ñor bay bëgg a xëm, geneen wàll gi ñor rekk."
+"Waaw lii nag ay néegi ñax la, néegi ñax bi ci kaw dañ kaa defar ko bam taaru lool. Bi ci des tamit ñu defar ko jël ab sars wërale ko ci kaw wutal kob buntu."
+"Nataal bii ngeen ma yónnee, gis naa ci biir nataal bi basaŋ yuñ ràbb, tegele leen. Boo xoolee, biñ leen ràbbe rafet nañ lool sax."
+"Waaw lii de ab ëe ay kii la ay pañe la, pañe ndugg.  Am na yu mboq, am na yu baxa ñu teg ko ci kaw lal walla kaw moket moket.. Xam naa ne ku ko jox jigéén dana si ndugg bu baax."
+"Lii nag dañ ciy faral di def ndugg maanaam jigéen ñi bu ñuy dem ja ba ëe, moom lañuy yor. Ñu koy woowe ci farañse pañe.. Dañu koy gàddu nag. Buñ ko gàddoo dañu ciy def lu mel niki ay ndugg daal lu mel ne jën, batañse,suppome, jën, naaje... Yooyu daal lañuy def ci biir."
+Lii ay toogukaayu mbedd la jardin public. Ñu ngi jagleel ñii di rombu ñii bëgg a noppaleeku ci ab gox mën nañu ñëw toog fii toog waxtaan.
+Waaw lii de ab nataal la boog. Maa ngi gis ñaari jigéen yu toog noonu yor ay leget yu dénk. Leget la yu dénk ak ay paan.
+"Lii na ñuy wax ""la tour la plus haute au Sénégal"", mi ngi nekk ""Monument  de la Renaissance Africaine"". Mi ngi nekk barab bi ñu bare di daje. Moo xam ñiy tàngal séen yaram, ñiy def i kõseer, moo xam ñiy def simnastig. Ñoom ñépp a fay daje. Palaas publig la boo xam ni bii ñoom ñépp a fay daje. Képp ku am xew-xew boo xam ni bii xaj na fa dangay ñów nga defe ko fa. Bu ko defee ñépp ñëw. Am na ci ñoo xam ni tàggat yaram moo leen fa aandi. Am na ñoo xam ni da ñuy jóge dëkk ak dëkk, ñów di ko wisite si. Monimaŋ bi, ñi ngi ko jëmale pàpp yaay ak doom. Doom jaa ngi joxoñ, baay ji téyee yaay ji ci ndigg li."
+"Lii duggukaay bu Luga la frontière bu Luga fii ngay dugg pour ñu wan la yaw mii nii yaa ngi dëkk biñ naan Luga. Luga nag ci Senegaal la bokk, ab lii la ci Senegaal."
+"Fii gis naa fi ñaari palaat yu weex. Ñaari palaat yu weex yi, benn bi am na cere. Benn bi, ñeexum kese la. Cere ji nag, ñeex na ñu ko, waaye ca wet ga gis naa ci ñaari bool yuy feeñ. Ñu ngi feeñ."
+"lii daal, am taax la mu gudd mu weex. bari lool ay ubbi féexal. ci kaw, juróom benni ubbi-féexal feeñ na fi. ci suuf, juróom benni ubbi-féexal feeñ na fi. ñu lal laltu bu xonq, ay nit yu sol lu xonq séq ko. waaw noo ngi ziyaar. yaang ci jàmm?"
+Waaw nataal bii nuru na ma ab li ñiy tudde ab tableau d'art mooy ñi nga xam ni dañ leen di defar daldi leen di bindaat daldi leen di dessiné mi ngi may jox benn waay it koo xam ni góor la may nirool ñu def ko bopp bi rëy lool sax gémmiñ gaa ngi ubbéeku mu am ay bëñ yu weex ca biir ay bët yu sew it bët yi tamit ubbéeku na mu am per bu ñuul kukk am ci wet gi ay lu weex nga gis doq gi mu xaw a nóox lool am ay am tamit ci ginnaaw mu mel ni dañ koo wat amaale lu ñuulaale ñu daldi ñuulal ginnaaw gi lool sax lool lool.
+"Nataal bii ma jot dee, Palais lay nuru waaye Palais bi nag am na ay garde corps yoo xam ne ñi ngi koy gardé, ba noppi mu am ñaari gaynde yoo xam ni ñoo ko séq. Palais bi nag paleb Senegaal lay nuru, ba noppi mu am ay xob yu wert ba noppi Palais bi bâtiment bi daa weex tàll."
+"Gis naa ni lii ab nataal la. Gis naa ci itam ay nit. Gis naa ci ku sol lu xonq.  gis naa ci ku sol lu ñuul. Gis naa ci ku sol mbaxane bu ñuul ak ay dàll yu ñuul. Gis naa itam ay xob ci ginnaaw;  xob yu ""vertes"". Gis naa itam ay ""grillage"" yu ñuul."
+Nataal bii de ñaari président la fii nag France la mu di président Maki Sàll ak Macron ñi ngi noon di dox ci affaire bu rouge bi am na oto yu topp seen gannaaw am na ñu taxaw seen wet yor lu mel ni ay fiil ay policier lañ Maki Sàll ak Macron ku nekk sol yére bu ñuul ak dàll yu ñuul kostin.
+"Ñii de ay góor lañu, ñu yéeg ci ay fas yor ay bant wutali fa ñu jëm."
+"Kale ngay séen njiitu réew mi la, Maki Sàll. Mi ngi nekk noonu ca kaw ndaamar ga, yékkati ay yoxoom. Leegi nag, polise yeek sàndarm si topp ko nii, toppekat yi di ko gàrde, war seeni ndaamar ñoom it. Njiitu réew mi moo nekk nee ca kow, ci ndaamar gu ñuul gi yékkati ay yoxoom."
+"Waaw nataal bi de waa ji war ab moto la. Genn góor gu yab ab moto. Am na ñu nekk ci wetam. Mu ngi sol Lu bula, lu bula..sol lu bula, moto bi am na lu weex am na lu ñuul.. Am na ñu sëgg nee sol yu xonq"
+"Ñii moom tamit ay takk-der nañ. Ñi ñi ngi bàyyikoo ci sàndarmëri nasiyonaal. Ñoom nag seen njariñ mooy di aar kaaraange njiitu réew mi. Ñu leen di tudde ci tubaab ""garde rouge, garde rapprochée ou garde présidentielle"". Ñoom la ñuy dénk wàllu kaaraange njiitu réew mi."
+"Nataal bi ab takk-der la waaw, yéeg ci benn móto waaw. Ci gannaawam yeneen takk-der yi ñi ngi taxaw di ko xool."
+"Ñenti alkaati, ñaar ñi yor seen bat ñow di àtte ñaar ñu nekk di xeex tëdd ci suuf. Mu am benn waay bu ta... Mu am benn taatu garab bu nekk ca wet ga, am benn waay bu ñëw."
+"Nataal bii maa ngi ci gis ay takk-der, ñu jàpp kenn ci doomi ndawi réewum Senegaal, di ko bunduxataal, di ko mettital, di ko dóor ba xaw koo tëddal ci suuf."
+"Lii nag ay pólisee yu jigéeñ la. Ëe bari nañu lool, ñu ràngoo ay fetal, sol ay gā yu weex, sol ay mbaxana, ëe, mbaxana mi tamit am na ay rëdd yu weex ci kaw, ak ay yére yu ñuul, jiital lu weex."
+aaa mooy li ñu waxoon rekk ñii ay policiers lañ ñoo xam ni da ñu ànd ak seen seef. ñoom la ñu gis ñu nekk taxaw nii. seen seef bi nag moo nekk nee. xay na nag da ñoo war a taxaw rekk coster les bras nii ci li ñu man a gis nii. mu nekk loo xam ne daal lu kii la daal.
+nataal bii Capitaine Séydinaa Omar Ture la. moo nekkoon section de recherche ñu radier ko.
+"nataal bii mu ngi nekk ak nit, nit ku xees, ku  toog ci baŋ bu ñuul wéerukaayu baŋ bi, ci fi ngay tegee sa bopp ni am ay bind yu weex. moom nag mu ngi tegle ñaari loxoom yi tiimale ko ci sikim bi. nit ki nak mu ngi sol yéreeg takk-der. maanaam yére bu am..."
+"Waaw lii de comme ni ma la ko waxe woon rekk, ceebu jën la, ceebu jën, ñu teg ko ciii, ci ci ci ci ci palaat bu weex,  teg lu mel ni ay ñambaan, daldi koy teg ci kow. Waaw boo xoolee ceeb bi, am na benn kuddu bu ñuul bu koy cubb, bu cubb ci kow nii, bu cubb cubb ñambaan ci kow."
+Juróom-ñetteelu nataal bi seetlu naa ci cere joo xam ne dañ koo defar. Biñ ko defaree bam noppi mu àndak ñeex moo xam ne moom lañ koy siife.
+Ba tey ñoo ngi gis ci nataal bii ay soldat soldat ñu ngi tollu ci fukkak ñaar ba fukkak ñett ci ay soldat soldat ya nga xam ne ñoom ñu ngi nekk ca palais ba kenn du laal palais bi ñoom nag duñ ree duñ kaf duñ muu dañuy mel ni ñu amul xol seen ginnaaw ñoo ngi gis buntu palais bi buntu palais bi rafet lool am benn tapis bu xonq am na ñaari gaynde ci ginnaaw am na ñeenti drapeau Senegaal ñoom nag ñoom ñoom noom waa Soldaar yi am na ñaari soldaar yoo xam ni ñi ngi sol seen i ay yére yu bula ak seen i tubéy yu ñuul bu fekkee ñi ci des nag ñoom ñoo bokk lenn lañ sol ñii sol seenum mbaxana mu xonq am lu noir am lu ñuul  ak seen i tubéy yu ñuul yor seen i ay yar ñee sol seen i gã ngir mën a gardé suñu suñu njiit bi.
+"Waaw ba tey lii de gis naa ne ag daamar, daamar gu taxaw la, ay sàndarma ñi ngi ci biir daamar ga, am na benn sàndarma bu nekk ca kanam. Waaw loolu de laa gis."
+"Waaw nataal bii tamit de gis naa ci benn bool bu weex ñu def ci caakiri def rasin ca kaw, teg ko nii ci suuf lool daal laa gis."
+"Kii de sennas ngóor la su taxaw si benn tali daldi yor aw xeer, takk masque. Xeer wi mu jëmale ko ca way-takk-der ya."
+"Nataal bii maa ngi ciy gis ay takk-der yu yor ay ngànnaay takk ay casque. Am na it góor gu tëdd nii, ñu mel ni ñu jëmsi ci moom."
+"Nataal bii nag, ay nit lañ yu taxaw di ñaxtu, maanaam di manifestewu, di jël ay xeef di leen sànnanteek takk-der yi. Am na nag ku nekk ci kanam nii solul maska, yor ay xeer di leen sànni. Mi ngi sol siletmaa bu weex ak tubéy ju ñuul. Mi ngi takk nag mbub lu xonq ci digg ginaaw gi."
+"Nataal bii maa ngi ciy gis ku yor raaya Senegaal. Daa mel ni ab nag doxu-ñaxtu la doonoon. Dagg lu mel ne tàndarma, garabu koko bu xonq, teg ko ci digg tali bi. Garabu koko. Ñii yore ""velo"", ñii dox ak seen tànk."
+"Waaw lii nag ab alkaati bu yor ngànnaayam, takk ndiggam di dàq ndaw sii di dem."
+"Gis naa fi ay takk-der yoo xam ni ñoo ngi solu seen i tenue, uuf seen i baaraas yoo xam ni dañuy def manifestation pour aar seen bopp ci baaraas bi."
+"Nataal bi ngeen ma yonnee, gis naa ni ay takk-der la. Takk der yoo xamantane bii nag, jël nañ loo xamantane bii mooy ay,... Ay baar. Ay verre yoo xam ne  dañ kaa defar pour ñu leen di baare, ñu jàkkarlook askan wi di sànnanteek ñoom ay xeer ak ay doj."
+"Waa ñii la nuy tuddee waay-takk -der yi. Ñii la ñuy tuddee sunu takk-der yi, ñooy waay takk-der yi. Pólise yi, ñoo taxaw nii daal dee takk ab... Daa dee sol benn mbaxanaam, daa di takk ab deram ci ndig li. Der gi nag dafa xaw a xonq tuuti. Mu ngi sol moom lu bula. Waaw... Daa di taxaw ci benn daamaar, mooy benn woto rekk. Benn woto la taxaw."
+"Lii de ab nataal la. Boo xoolee yaa ngi gis lu mel ne ay kàdd lañ si def, ay matt yun dagg moom la gën a niru.Waaw! Boo xoole am na ay wirgo yu nëtëx yuy feeñ ci ginnaaw, wirgo yu nëtëx yuy feeñ ci ginnaaw. Xaw naa niru ab ndawal, xaw naa niroo itam ab ndawal."
+Màndarga bi de;  uhuum!  dafa mel ne ni ma ko gise yàpp la.  Xaw ma yàpp nag xaw ma yàppu xar lay doon waaye yàpp la. Teg ko ci kayit ; ci kayit lañ ko laxas. Waawaw! Tabarkalla! Maasàllaa! yàpp bi moom neex naa defar lool de.
+"Haa lii de moom, nataal bii may séen moom koŋ fuule laa ko teg. Koŋ Fuule de moom ku ko def suppu-kànja moom, ku ko togg suppu kànja bu neex, def sa sippax ak sa  yépp di nga am suppu kànja bu neex boo tegal jaxate-jaxate yi."
+"Lii de lanuy tuddee jën, keccax, keccax. Ñu defar ko ba mu pare def si nataal."
+"Lii ma gis nii de : aw yàpp wun làkk la. Waaye nag sooy xool jëmm yàpp wi day mel ni ginaar lay niru. Ñu ngi ko def ci aw kayit, dunq wi dangay gis ay karawi karaw"
+"Nataal bii ma jot nag, lii nag daa ndiru yàpp. Yàpp wum mën a donn nag, waaye yàpp woo xam ni daal dañ mel ni ku ko làkk daal lay xaw a ndiru. xam naa ñi daal yàpp lañ koy wax."
+"Waaw foto bi de, ñu ngi ciy gis benn palaat bu weex boo xam ne si dafa def laax. Laaxu araw. Laax yi nag nu ngi ko cifee cif boo xam ne sii nag cifaayu nëtëri la..."
+"Ëe waaw bii ay takk-der la. Takk-der yi  ñu jiital Guy Maris Saañaa. Moom lañu jiital nii, jàpp ko di ko jiital di ko yóbb waaw. Am na taskatu-xibaar bu nekk nële ca wet ga sol yére bu weex, bu bula ak tubéy bu ñuul."
+Nataal bii day wane ndungu siin bu nekk ca Ndakaaru ñu koy woowe rëbes. Booy xool ci buntu bi am na ay nit yu fa taxaw ak ay alkaati.
+"Lii ab nataal la buy wone benn béréb bu tudd ""Maison d'arrêt de de correction bu rëbës"". Rëbës nag fii ci Senegaal la nekk. Gis nanu ci bunt bu wert ak ay alkaati yu taxaw ci bunt bi. Gis nanu fi ay nit ñu bari. Ñenn ñi sol  yére yu bula, ñeneen ci ñu sol yére yu ñuul."
+"Maa ngi gis ci nataal bi benn buntu, benn buntu bu pharmacie bu nice bu vert mooy yooyu di nekk ci buntu muy benn gendarmerie..."
+Waaw mbokk mi nataal bii de bi ma xoolee gis naa ci ay góor ñu taxaw ñu yékkati seen i loxo jàkkarloo ak stade bi gis naa ci ku yékkati drapeau Senegaal bi ca kaw.
+Lii ab barabu bu xalaatukaayu xam-xam la. Ñu gis ay nit ñu toog ci seeni buro di sëg ci seeni ordinaatëer. Buro yi nag ñu ngi am ay kulëer yu baxa ak ay làmp yu tàkk ci kow.
+"Nataal bii nag moom, ag jigéen a ci nekk yor ag cameraa teg ko ci bët yi, di xool talal loxo bi. Am na lu ci nekk lu mel ni sãru lënd bulo am ay... Ay làmp ci koñ yi."
+"Waaw nataal bii, nataal la boob as ndaw moo fi nekk jël ay am mbir teg ko ci ay gëtam yékkati ab yoxoom. Am na tamit mbir mu baxa loo xam ne mi ngi ko wër mel ni ag lënd."
+"Nataal bi ngeen ma yónnee mooy juróom fukkeelu nataal bi ak ñaar moom ngeen ma yónneewoon léegi nii. Tontu naa ci waaye li ma ci gis mooy am jumtukaay la boo xam ne ci wàllu ""internet"" la jëm te jumtukaay boobu te moom lañ naan ""réseau"" bi nga xam ne mooy tigo te mu soppeeku nekk free moom la. Am na am mbind mu xonq ca kaw. Moom lu ñuul la dafa ñuul, moom jumtukaay bi am lu xonq ci kaw, am tamit ay bësukaay."
+"Nataal bii nag gis nan ci ndox maanaam dex . Nit ñi toog ci digg dex gi, ëe... Di naan. Seeni, ëe... Di noos, di noos. Gis nan si ay góor ak ay jigéen. Gis nan sii... benn góor moomit moom mu tée doomam di ko nax yaayam mu ngi leen di, mu ngi leen di nataal. Mu leen di dóor nataal. Moom mu ngi sol yére bu ñuul ni ibaadu yi di soloo. Ñi ci des ñoom , ñu ngi toog di waxtaan di naan ndox."
+"Waaw nataal bi ñi ngi ci gis ay jëmmu yu toog, ñu tiim seen ay jumtukaay, jumtukaay ak ñoom seen."
+"Nataal bii ngeen ma yónnee de centre bu mag a mag la boo xam ne jigéen ñi ak góor ñi ñépp ñoo ngi cib liggéey, ku nekk yaa ngi tiim sab ordinateur di liggéey."
+"Nataal bii maa ngi ciy gis ab ordinatéer manaam ab nosukaay, nosukaay laa ci gis mooy ordinatéer moom laa ci gis ñu taal ko. Moom rekk laa ci gis."
+"Li ñi jot a gis ab ""tampon"" la walla sax ñu naan ko ""cachet"". ""Cachet"" nag  dafa nekk loo xamente ni moom lañii jël ngir màndargaal yoo xamente ne ay nit ñoo ko bokk walla sax foo xamante ne kenn nit moo ko moom. Loolu nag ""tampon"" bu nekk am na fu mu jóge am na lu muy wone ci bataaxal buñ ko def"
+"Lii ab torlu la, maanaam njëfandikukaayu torlu la. Ag jëfekaay la boo xam ni bii mën. nga cee def sab torlu. Torlu nag mooy màndarga ginnaaw boo bindee ab bataaxal ci misaal, ca mujj ga nga def ci sa torlu. Loolu day wone ni yow kat bataaxal bii yaa ko bind. Walla bu fekkee da ngaa séq ak nit, nga jël sa torlu def ca, day wone ni yow kat yaa séq ak nit ki lii ci ca jëmm. Kon torlu lu am a am solo la. Lii nag ab jëfandikukaat la boo xam ni mën nga cee def sa torlu bopp Boo xam ni da ngay bës rek my daldi tàmpe ci kayit gi, daldi ci sax. Mu nekk sa màndarga."
+Waaw nataal bii dafay wone ab masin bu ndaw boo xam ni dan cii bind waxtu . Masin bi daa mel ni xool waxtu moo ko tax a jóg. Waayit mënees na ko jifandikoo xéy na ci generen waaw..
+"Benn montar la rekk, benn montar. Witëer taransiis moo jot, waaw witëer taransiis moo jot, benn montar la bu weex. Benn montar bu weex la, waaw."
+"Waaw nataal bii de jumtukaay la. Jumtukaay bi melo bi dafa xonq, ci biir ëe lu xonq li lu weex moo nekk ci biir. Am na ay joxoñ yu ñuul waaw. Mu mel ne ""façon"" tele."
+"Gis naa fi benn affaire bu am ñaari couleur. Couleur bu xonq ak couleur bu weex. Ci suuf ñi ngi ci bind presse, mbind mu ñuul presse."
+"Waaw lii nag jumtukaay la bu xereñ boo xam ni ni mën nan cee nattee, xaymaa mbir mu mu mën a doon ; lu mel ne cee, ceeb  walla dugub walla suukër, walla lu ni mel mën ka cee natt ba xayma ab tolluwaayam."
+"Waaw nataal bii ag xarala la goo xamente ne ""écran"" la bu mag ci saalu sinemaa, nit ñi toog di seetaan, nit ñu bari. Am na ku toog ci puus-puus, am ñu taxaw. Am na xale bu ci nekk."
+Lii de ab jollasu bu ñu tijji. Ki ko tijji nag nuru na ab defarkat ngir saytu lan mooy sikk si.
+"Lii ab palaas bu nu tëj la aa, dessin nanu si benn nit muy niroo ak nit ku weex, takk ay weer ag genn jollasu ci wetu ndeyjoor gi. Palaas bi nag palaasu defarkatu telefon la may, defarkatu jollasu la may nirool. Waa ñu peinture ko peinture bu ñuul, peinture boo xam ne ay kulëer la. Mu mel ne gis naa si ñuul, gis naa si kulëer bu xonq, gis naa si tamit kulëer bu soon."
+Nataal bii gis naa ci luñ bind télécentre......
+"Nataal bii ab tele moo ci nekk bu ànd ak rojo. Mi ngi am ci digg bi lu verte. Am na ay biton yu nekk ci wet gi, ñaari biton. Am na tamit ab ""baffle"" bu nekk ci suuf. Am na tamit ñaari NTN yu taxaw."
+Waaw liñ seetlu si nataal bi nag ab radio la radio bi nag am na ñaari xëccukaay kaay si kaw loolu lañ seetlu si nataal bi ab radio la daal.
+"Lii ngeen gis nag ci nataal bi palaa la boo xam ne saa senegaal yi ñoo ëpp li ñu koy togg muy domodaa. Am na limoŋ ci kaw, am na kaani, am na bulet, am na pombiteer, am na karoot, am na suppome. Lu weex li nag mooy ñànkatañ bi."
+Kii senn waay la di yor ub telefon. Moom nag mi ngi ko téye ak ñaari yoxoom di ci fo.
+"Lii benn nit la boo xam ne dafa yor lu ñuy fowe foyukaay, foyukaay la mi ngi tuddu PES"
+Nataal bii de ag jollasu la gu xaw a ñuul am ay bitoŋ ñaari mbind am mbind mu ñuul ci kaw. Mu ngi nekk ñaari jollasu daal yu tege ci suuf.  Am na lu mel ni ag kees gu tege ci wetam gu ñuul.
+"Waaw lii mën nanu wax ay jumtykaay la ; jumtukaay yu bees yi boo xam ne soo tàngee danga koy taal nga sedd guyy waaw. Ci ay jumtukaay yu bees la, ba mu xewee sax yàggul."

wolof-translate/wolof_translate/utils/database_manager.py

@@ -0,0 +1,113 @@
+from pymongo.mongo_client import MongoClient
+from pymongo.server_api import ServerApi
+import pandas as pd
+
+class TranslationMongoDBManager:
+    
+    def __init__(self, uri: str, database: str):
+        
+        # recuperate the client
+        self.client = MongoClient(uri)
+        
+        # recuperate the database
+        self.db = self.client.get_database(database)
+        
+    def insert_documents(self, documents: list, collection: str = "sentences"):
+        
+        # insert documents inside a collection
+        results = self.db[collection].insert_many(documents)
+        
+        return results
+    
+    def insert_document(self, document: dict, collection: str = "sentences"):
+        
+        assert '_id' in document
+        
+        # get the id of the last sentence (recuperate the max id and add 1 to it)
+        max_id = self.get_max_id(collection)
+        
+        # add the new sentences
+        document['_id'] = max_id
+        
+        results = self.db[collection].insert_one(
+            document
+        )
+        
+        return results
+    
+    def update_document(self, id: int, document: dict, collection: str = "sentences"):
+        
+        # update the document
+        results = self.db[collection].update_one(
+            {
+                '_id': {
+                    '$eq': id
+                }
+            },
+            {
+                '$set': document
+            }
+        )
+        
+        return results
+    
+    def delete_document(self, id: int, document: dict, collection: str = "sentences", del_collection: str = "deleted"):
+        
+        # recuperate the document to delete
+        del_sent = self.db[collection].find_one(
+            {
+                '_id': {
+                    '$eq': id
+                }
+            }
+        )
+        
+        # delete the sentence
+        self.db[collection].delete_one(
+            {
+                '_id': {'$eq': del_sent['_id']}
+            }
+        )
+        
+        # add the sentences to the deleted sentences
+        del_sent['_id'] = len(list(deleted.find()))
+        
+        results = self.db[del_collection].insert_one(
+            del_sent
+        )
+        
+        return results
+    
+    def get_max_id(self, collection):
+        
+        # recuperate the maximum id
+        id = list(self.db[collection].find().sort('_id', -1).limit(1))[0]['_id']
+        
+        return id
+    
+    def save_data_frames(self, sentences_path: str, deleted_path: str, collection: str = "sentences", del_collection: str = "deleted"):
+        
+        # recuperate the new corpora
+        new_corpora = pd.DataFrame(list(self.db[collection].find()))
+
+        # recuperate the deleted sentences as a Data Frame
+        deleted_df = pd.DataFrame(list(self.db[del_collection].find()))
+
+        # save the data frames as csv files
+        new_corpora.set_index('_id', inplace=True)
+
+        deleted_df.set_index('_id', inplace=True)
+
+        new_corpora.to_csv(sentences_path, index=False)
+
+        deleted_df.to_csv(deleted_path, index=False)
+    
+    def load_data_frames(self, collection: str = "sentences", del_collection: str = "deleted"):
+        
+        # recuperate the new corpora
+        new_corpora = pd.DataFrame(list(self.db[collection].find()))
+
+        # recuperate the deleted sentences as a Data Frame
+        deleted_df = pd.DataFrame(list(self.db[del_collection].find()))
+        
+        return new_corpora, deleted_df