Uploading notebooks for replication of the training

training_notebooks/01_Downloading_and_preprocessing_corpora.ipynb

@@ -0,0 +1,1100 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cltk"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from cltk.data.fetch import FetchCorpus"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cltk.utils.get_cltk_data_dir()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_downloader = FetchCorpus(language=\"lat\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['lat_text_perseus',\n",
+       " 'lat_treebank_perseus',\n",
+       " 'lat_text_latin_library',\n",
+       " 'phi5',\n",
+       " 'phi7',\n",
+       " 'latin_proper_names_cltk',\n",
+       " 'lat_models_cltk',\n",
+       " 'latin_pos_lemmata_cltk',\n",
+       " 'latin_treebank_index_thomisticus',\n",
+       " 'latin_lexica_perseus',\n",
+       " 'latin_training_set_sentence_cltk',\n",
+       " 'latin_word2vec_cltk',\n",
+       " 'latin_text_antique_digiliblt',\n",
+       " 'latin_text_corpus_grammaticorum_latinorum',\n",
+       " 'latin_text_poeti_ditalia',\n",
+       " 'lat_text_tesserae',\n",
+       " 'cltk_lat_lewis_elementary_lexicon']"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "corpus_downloader.list_corpora"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_downloader.import_corpus(\"lat_text_perseus\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_downloader.import_corpus(\"lat_text_latin_library\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_downloader.import_corpus(\"lat_text_tesserae\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "corpus_downloader.import_corpus(\"latin_text_corpus_grammaticorum_latinorum\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cltk_data_dir = cltk.utils.get_cltk_data_dir()\n",
+    "\n",
+    "latin_library_path = os.path.join(cltk_data_dir, \"lat\", \"text\", \"lat_text_latin_library\")\n",
+    "perseus_path = os.path.join(cltk_data_dir, \"lat\", \"text\",  \"lat_text_perseus\", \"cltk_json\")\n",
+    "tesserae_path = os.path.join(cltk_data_dir, \"lat\", \"text\",  \"lat_text_tesserae\", \"texts\")\n",
+    "corpus_grammaticorum_path = os.path.join(cltk_data_dir, \"lat\", \"text\", \"latin_text_corpus_grammaticorum_latinorum\") "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "my_corpus_path = \"custom_latin_corpus\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import shutil"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not os.path.exists(my_corpus_path):\n",
+    "    os.mkdir(my_corpus_path)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Processing Latin Library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# the latin library ihas files already in .txt format, so we just need to copy them"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_latin_library_path = os.path.join(my_corpus_path, \"latin_library\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    shutil.copytree(\n",
+    "        latin_library_path,\n",
+    "        new_latin_library_path\n",
+    "    )\n",
+    "except:\n",
+    "    pass"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Processing Perseus Library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_perseus_path = os.path.join(\n",
+    "    my_corpus_path, \"perseus_library\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not os.path.exists(new_perseus_path):\n",
+    "    os.mkdir(new_perseus_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['terence__the-self-tormenter__english.json',\n",
+       " 'plautus-titus-maccius__trinummus__latin.json',\n",
+       " 'ausonius-decimus-magnus__griphus-ternarii-numeri__latin.json',\n",
+       " 'cicero__in-defense-of-aulus-cluentius-habitus__latin.json',\n",
+       " 'plautus-titus-maccius__bacchides__latin.json',\n",
+       " 'ovid__remedy-of-love__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-ieiunio-adversus-psychicos__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__ad-uxorem__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__antoninus-caracalla__latin.json',\n",
+       " 'cicero__on-behalf-of-marcus-tullius__english.json',\n",
+       " 'ausonius-decimus-magnus__versus-paschales-prosodic__latin.json',\n",
+       " 'claudianus-claudius__panegyricus-de-sexto-consulatu-honorii-augusti__latin.json',\n",
+       " 'nepos-cornelius__agesilaus__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__apologeticum__latin.json',\n",
+       " 'paris-julius__facta-et-dicta-memorabilia__latin.json',\n",
+       " 'cicero__de-optimo-genere-oratorum__latin.json',\n",
+       " 'cicero__paradoxa-stoicorum__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-titus__latin.json',\n",
+       " 'virgil__eclogues__latin.json',\n",
+       " 'martial__epigrammata__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-carnis-resurrectione__latin.json',\n",
+       " 'cicero__for-marcus-fonteius__latin.json',\n",
+       " 'cicero__in-the-senate-after-his-return__latin.json',\n",
+       " 'terence__the-brothers__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__pertinax__latin.json',\n",
+       " 'nepos-cornelius__datames__latin.json',\n",
+       " 'nepos-cornelius__cato__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-constantia__latin.json',\n",
+       " 'tibullus__elegiae__english.json',\n",
+       " 'celsus-aulus-cornelius__de-medicina__english.json',\n",
+       " 'ovid__letters-from-the-black-sea__latin.json',\n",
+       " 'curtius-rufus-quintus__historiarum-alexandri-magni__latin.json',\n",
+       " 'caesar-julius__gallic-war__english.json',\n",
+       " 'ausonius-decimus-magnus__epigrammaton-liber__latin.json',\n",
+       " 'cicero__letters-to-his-friends__latin.json',\n",
+       " 'claudianus-claudius__in-rufinum__latin.json',\n",
+       " 'gellius-aulus__noctes-atticae__latin.json',\n",
+       " 'seneca-lucius-annaeus-55-bc-ca-39-ad__fragmenta__latin.json',\n",
+       " 'cicero__four-orations-against-lucius-catilina__english.json',\n",
+       " 'cicero__de-finibus-bonorum-et-malorum__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__caligula__latin.json',\n",
+       " 'cicero__letters-to-his-brother-quintus__latin.json',\n",
+       " 'nepos-cornelius__pausanias__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__carus-et-carinus-et-numerianus__latin.json',\n",
+       " 'cicero__for-sulla__latin.json',\n",
+       " 'plautus-titus-maccius__miles-gloriosus__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__oedipus__latin.json',\n",
+       " 'horace__satires__english.json',\n",
+       " 'seneca-lucius-annaeus-55-bc-ca-39-ad__suasoriae__latin.json',\n",
+       " 'cicero__for-archias__latin.json',\n",
+       " 'columella-lucius-junius-moderatus__res-rustica__latin.json',\n",
+       " 'plautus-titus-maccius__asinaria__english.json',\n",
+       " 'seneca-lucius-annaeus-55-bc-ca-39-ad__controversiae__latin.json',\n",
+       " 'plautus-titus-maccius__captivi__english.json',\n",
+       " 'cicero__for-publius-quinctius__english.json',\n",
+       " 'cicero__for-marcus-fonteius__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__firmus-saturninus-proculus-et-bonosus__latin.json',\n",
+       " 'cicero__in-defense-of-l-murena__latin.json',\n",
+       " 'ausonius-decimus-magnus__epitaphia__latin.json',\n",
+       " 'plautus-titus-maccius__amphitruo__english.json',\n",
+       " 'cicero__for-marcus-caelius__latin.json',\n",
+       " 'claudianus-claudius__panegyricus-de-quarto-consulatu-honorii-augusti__latin.json',\n",
+       " 'nepos-cornelius__timotheus__latin.json',\n",
+       " 'claudianus-claudius__panegyricus-de-tertio-consulatu-honorii-augusti__latin.json',\n",
+       " 'plautus-titus-maccius__persa__english.json',\n",
+       " 'plautus-titus-maccius__truculentus__latin.json',\n",
+       " 'nepos-cornelius__alcibiades__latin.json',\n",
+       " 'lucan__civil-war__latin.json',\n",
+       " 'minucius-felix-marcus__octavius__latin.json',\n",
+       " 'ausonius-decimus-magnus__ephemeris__latin.json',\n",
+       " 'ovid__metamorphoses__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__apocolocyntosis__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-brevitate-vitae__latin.json',\n",
+       " 'claudianus-claudius__de-bello-gothico__latin.json',\n",
+       " 'cicero__orations-against-verres__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-vespasianus__english.json',\n",
+       " 'plautus-titus-maccius__epidicus__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__agamemnon__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__adversus-marcionem__latin.json',\n",
+       " 'claudianus-claudius__epithalamium-de-nuptiis-honorii-augusti__latin.json',\n",
+       " 'cicero__letters-to-brutus__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__tiberius__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__antoninus-geta__latin.json',\n",
+       " 'cicero__in-defense-of-lucius-flaccus__english.json',\n",
+       " 'horace__satires__latin.json',\n",
+       " 'ovid__ibis__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__verus__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-idolatria__latin.json',\n",
+       " 'apuleius__apologia__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-testimionio-animae__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-praescriptionibus-hereticorum__latin.json',\n",
+       " 'ovid__epistulae__english.json',\n",
+       " 'seneca-lucius-annaeus__de-tranquilitate-animi__latin.json',\n",
+       " 'cicero__in-defense-of-aulus-cluentius-habitus__english.json',\n",
+       " 'seneca-lucius-annaeus__de-consolatione-ad-helviam__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__maximus-et-balbinus__latin.json',\n",
+       " 'tacitus-cornelius__agricola__latin.json',\n",
+       " 'horace__odes__latin.json',\n",
+       " 'boethius-d-524__quomodo-trinitas-unus-deus-ac-non-tres-dii-de-trinitate__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__domitianus__english.json',\n",
+       " 'claudianus-claudius__carminum-minorum-corpusculum__latin.json',\n",
+       " 'ovid__sorrows__latin.json',\n",
+       " 'nepos-cornelius__phocion__latin.json',\n",
+       " 'tacitus-cornelius__germania__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__gallieni-duo__latin.json',\n",
+       " 'caesar-julius__gallic-war__latin.json',\n",
+       " 'cicero__orator__latin.json',\n",
+       " 'cicero__de-republica__latin.json',\n",
+       " 'ovid__epistulae__latin.json',\n",
+       " 'cicero__de-fato__latin.json',\n",
+       " 'nepos-cornelius__de-regibus__latin.json',\n",
+       " 'sallust__historiae__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__tacitus__latin.json',\n",
+       " 'plautus-titus-maccius__mostellaria__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__valeriani-duo__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-virginibus-velandis__latin.json',\n",
+       " 'ausonius-decimus-magnus__gratiarum-actio__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__antoninus-pius__latin.json',\n",
+       " 'nepos-cornelius__iphicrates__latin.json',\n",
+       " 'ausonius-decimus-magnus__genethliacon-ad-ausonium-nepotem__latin.json',\n",
+       " 'plautus-titus-maccius__rudens__english.json',\n",
+       " 'nepos-cornelius__pelopidas__latin.json',\n",
+       " 'plautus-titus-maccius__aulularia__english.json',\n",
+       " 'cicero__to-the-citizens-after-his-return__english.json',\n",
+       " 'seneca-lucius-annaeus-plays__hercules-oetaeus__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-baptismo__latin.json',\n",
+       " 'boethius-d-524__liber-de-persona-et-duabus-naturis-contra-eutychen-et-nestorium__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-claudius__latin.json',\n",
+       " 'nepos-cornelius__aristides__latin.json',\n",
+       " 'terence__the-eunuch__english.json',\n",
+       " 'cicero__academica__latin.json',\n",
+       " 'terence__the-eunuch__latin.json',\n",
+       " 'plautus-titus-maccius__bacchides__english.json',\n",
+       " 'celsus-aulus-cornelius__de-medicina__latin.json',\n",
+       " 'cicero__tusculanae-disputationes__latin.json',\n",
+       " 'cicero__timaeus__latin.json',\n",
+       " 'nepos-cornelius__chabrias__latin.json',\n",
+       " 'sallust__bellum-iugurthinum__latin.json',\n",
+       " 'boethius-d-524__de-fide-catholica__latin.json',\n",
+       " 'pliny-the-elder__naturalis-historia__latin.json',\n",
+       " 'nepos-cornelius__eumenes__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__diadumenus-antoninus__latin.json',\n",
+       " 'cicero__divinatio-against-q-caecilius__latin.json',\n",
+       " 'plautus-titus-maccius__pseudolus__english.json',\n",
+       " 'seneca-lucius-annaeus-plays__medea__latin.json',\n",
+       " 'plautus-titus-maccius__menaechmi__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__caligula__english.json',\n",
+       " 'juvenal__satires__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__divus-claudius__latin.json',\n",
+       " 'lucan__civil-war__english.json',\n",
+       " 'cicero__de-natura-deorum__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__helius__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-spectaculis__latin.json',\n",
+       " 'plautus-titus-maccius__cistellaria__english.json',\n",
+       " 'cicero__against-publius-servilius-rullus__english.json',\n",
+       " 'cicero__brutus__latin.json',\n",
+       " 'claudianus-claudius__panegyricus-dictus-probino-et-olybrio-consulibus__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__marcus-antoninus-philosophus__latin.json',\n",
+       " 'terence__phormio__english.json',\n",
+       " 'ausonius-decimus-magnus__praefatiunculae__latin.json',\n",
+       " 'cicero__de-amicitia__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__scorpiace__latin.json',\n",
+       " 'horace__ars-poetica__english.json',\n",
+       " 'propertius-sextus__elegies__latin.json',\n",
+       " 'tibullus__elegiae__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-cultu-feminarum__latin.json',\n",
+       " 'terence__the-mother-in-law__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__maximini-duo__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-consolatione-ad-polybium__latin.json',\n",
+       " 'statius-p-papinius-publius-papinius__silvae__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__ad-martyras__latin.json',\n",
+       " 'plautus-titus-maccius__asinaria__latin.json',\n",
+       " 'virgil__aeneid__english.json',\n",
+       " 'seneca-lucius-annaeus-plays__de-clementia__latin.json',\n",
+       " 'plautus-titus-maccius__curculio__english.json',\n",
+       " 'plautus-titus-maccius__aulularia__latin.json',\n",
+       " 'cicero__de-officiis__latin.json',\n",
+       " 'ovid__book-of-days__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__avidius-casius__latin.json',\n",
+       " 'vitruvius-pollio__on-architecture__latin.json',\n",
+       " 'terence__phormio__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__de-vita-hadriani__latin.json',\n",
+       " 'virgil__eclogues__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__opilius-macrinus__latin.json',\n",
+       " 'cicero__on-pompeys-command__latin.json',\n",
+       " 'claudianus-claudius__panegyricus-dictus-manlio-theodoro-consuli__latin.json',\n",
+       " 'ausonius-decimus-magnus__precationes__latin.json',\n",
+       " 'ovid__metamorphoses__english.json',\n",
+       " 'ammianus-marcellinus__rerum-gestarum__english.json',\n",
+       " 'cicero__orations-against-verres__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__divus-aurelianus__latin.json',\n",
+       " 'sallust__catilinae-coniuratio__english.json',\n",
+       " 'cicero__in-defense-of-caius-rabirius__latin.json',\n",
+       " 'cicero__partitiones-oratoriae__latin.json',\n",
+       " 'plautus-titus-maccius__curculio__latin.json',\n",
+       " 'ovid__amores__english.json',\n",
+       " 'titus-livius-livy__the-history-of-rome__english.json',\n",
+       " 'terence__andria__latin.json',\n",
+       " 'cicero__for-archias__english.json',\n",
+       " 'seneca-lucius-annaeus-plays__hercules-furens__latin.json',\n",
+       " 'ausonius-decimus-magnus__mosella__latin.json',\n",
+       " 'cicero__for-quintus-roscius-the-actor__english.json',\n",
+       " 'valerius-flaccus-gaius__argonautica__latin.json',\n",
+       " 'prudentius-b-348__contra-orationem-symmachia__latin.json',\n",
+       " 'vitruvius-pollio__on-architecture__english.json',\n",
+       " 'cicero__letters-to-atticus__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__probus__latin.json',\n",
+       " 'plautus-titus-maccius__rudens__latin.json',\n",
+       " 'horace__odes__english.json',\n",
+       " 'catullus__carmina__english.json',\n",
+       " 'cicero__for-sextus-roscius-of-ameria__latin.json',\n",
+       " 'ausonius-decimus-magnus__epicedion-in-patrem__latin.json',\n",
+       " 'nepos-cornelius__dion__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__phaedra__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-claudius__english.json',\n",
+       " 'prudentius-b-348__dittochaeon__latin.json',\n",
+       " 'ovid__amores__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__octavia__latin.json',\n",
+       " 'cicero__for-quintus-roscius-the-actor__latin.json',\n",
+       " 'apuleius__metamorphoses__latin.json',\n",
+       " 'ausonius-decimus-magnus__oratio-versibus-rhopalicis__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-julius__english.json',\n",
+       " 'plautus-titus-maccius__persa__latin.json',\n",
+       " 'cicero__four-orations-against-lucius-catilina__latin.json',\n",
+       " 'cicero__on-behalf-of-marcus-tullius__latin.json',\n",
+       " 'ausonius-decimus-magnus__technopaegnion__latin.json',\n",
+       " 'plautus-titus-maccius__mercator__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-consolatione-ad-marciam__latin.json',\n",
+       " 'quintus-tullius-cicero__commentariolum-petitionis__latin.json',\n",
+       " 'ovid__art-of-love__english.json',\n",
+       " 'nepos-cornelius__themistocles__latin.json',\n",
+       " 'cicero__de-divinatione__latin.json',\n",
+       " 'claudianus-claudius__de-bello-gildonico__latin.json',\n",
+       " 'nepos-cornelius__atticus__latin.json',\n",
+       " 'terence__the-brothers__english.json',\n",
+       " 'tertullian-ca-160-ca-230__de-pallio__latin.json',\n",
+       " 'plautus-titus-maccius__captivi__latin.json',\n",
+       " 'ausonius-decimus-magnus__eclogarum-liber__latin.json',\n",
+       " 'ausonius-decimus-magnus__liber-protrepticus-ad-nepotem__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-oratione__latin.json',\n",
+       " 'ausonius-decimus-magnus__caesares__latin.json',\n",
+       " 'sallust__bellum-iugurthinum__english.json',\n",
+       " 'plautus-titus-maccius__epidicus__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__antoninus-heliogobalus__latin.json',\n",
+       " 'cicero__for-publius-quinctius__latin.json',\n",
+       " 'boethius-d-524__quomodo-substantiae-in-eo-quod-sint-bonae-sint-cum-non-sint-substanialia-bona__latin.json',\n",
+       " 'nepos-cornelius__hannibal__latin.json',\n",
+       " 'cicero__in-the-senate-after-his-return__english.json',\n",
+       " 'cicero__on-behalf-of-aulus-caecina__english.json',\n",
+       " 'ausonius-decimus-magnus__bissula__latin.json',\n",
+       " 'terence__the-mother-in-law__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-monogamia__latin.json',\n",
+       " 'prudentius-b-348__hamartigenia__latin.json',\n",
+       " 'plautus-titus-maccius__poenulus__english.json',\n",
+       " 'horace__epistulae__latin.json',\n",
+       " 'plautus-titus-maccius__menaechmi__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__alexander-severus__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__adversus-hermogenem__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-vita-beata__latin.json',\n",
+       " 'gellius-aulus__noctes-atticae__english.json',\n",
+       " 'seneca-lucius-annaeus-plays__apocolocyntosis__english.json',\n",
+       " 'tertullian-ca-160-ca-230__ad-nationes-libri-duo__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-patientia__latin.json',\n",
+       " 'claudianus-claudius__fescinnina-de-nuptiis-honorii-augusti__latin.json',\n",
+       " 'cicero__on-oratory__latin.json',\n",
+       " 'cicero__de-senectute__latin.json',\n",
+       " 'nepos-cornelius__conon__latin.json',\n",
+       " 'horace__ars-poetica__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-carne-christi__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__thyestes__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__vitellius__english.json',\n",
+       " 'prudentius-b-348__liber-peristephanon__latin.json',\n",
+       " 'virgil__aeneid__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__nero__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__tiberius__english.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__didius-julianus__latin.json',\n",
+       " 'ovid__remedy-of-love__english.json',\n",
+       " 'lucretius__de-rerum-natura__english.json',\n",
+       " 'plautus-titus-maccius__trinummus__english.json',\n",
+       " 'plautus-titus-maccius__mostellaria__english.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-augustus__english.json',\n",
+       " 'ovid__art-of-love__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__troades-furens__latin.json',\n",
+       " 'horace__carmen-saeculare__latin.json',\n",
+       " 'virgil__georgics__latin.json',\n",
+       " 'ovid__art-of-beauty__latin.json',\n",
+       " 'claudianus-claudius__de-raptu-proserpinae__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__severus__latin.json',\n",
+       " 'prudentius-b-348__apotheosis__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-pudicitia__latin.json',\n",
+       " 'plautus-titus-maccius__casina__latin.json',\n",
+       " 'nepos-cornelius__lysander__latin.json',\n",
+       " 'prudentius-b-348__praefetio__latin.json',\n",
+       " 'cicero__de-inventione__latin.json',\n",
+       " 'nepos-cornelius__thrasybulus__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__galba__latin.json',\n",
+       " 'cicero__for-sulla__english.json',\n",
+       " 'ausonius-decimus-magnus__ordo-urbium-nobilium__latin.json',\n",
+       " 'cicero__on-pompeys-command__english.json',\n",
+       " 'silius-italicus-tiberius-catius__punica__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-corona__latin.json',\n",
+       " 'sallust__catilinae-coniuratio__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__adversus-valentinianos__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__commodus-antoninus__latin.json',\n",
+       " 'ausonius-decimus-magnus__epistulae__latin.json',\n",
+       " 'nepos-cornelius__hamilcar__latin.json',\n",
+       " 'plautus-titus-maccius__mercator__english.json',\n",
+       " 'plautus-titus-maccius__miles-gloriosus__english.json',\n",
+       " 'prudentius-b-348__epilogus__latin.json',\n",
+       " 'apuleius__florida__latin.json',\n",
+       " 'lucretius__de-rerum-natura__latin.json',\n",
+       " 'ausonius-decimus-magnus__de-herediolo__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-julius__latin.json',\n",
+       " 'horace__epodi__latin.json',\n",
+       " 'seneca-lucius-annaeus-55-bc-ca-39-ad__excerpta-controversiae__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__tyranni-triginta__latin.json',\n",
+       " 'cicero__to-the-citizens-after-his-return__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-ira__latin.json',\n",
+       " 'plautus-titus-maccius__casina__english.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-titus__english.json',\n",
+       " 'prudentius-b-348__psychomachia__latin.json',\n",
+       " 'seneca-lucius-annaeus-plays__phoenissae__latin.json',\n",
+       " 'cicero__in-defense-of-caius-rabirius__english.json',\n",
+       " 'plautus-titus-maccius__truculentus__english.json',\n",
+       " 'suetonius-ca-69-ca-122__otho__latin.json',\n",
+       " 'cicero__topica__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-providentia__latin.json',\n",
+       " 'terence__the-self-tormenter__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__nero__english.json',\n",
+       " 'ausonius-decimus-magnus__parentalia__latin.json',\n",
+       " 'claudianus-claudius__de-consulatu-stilichonis__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__vitellius__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-exhortatione-castitatis-liber__latin.json',\n",
+       " 'boethius-d-524__utrum-pater-et-filius-ac-spiritus-sanctus-de-divinitate-substantialiter-praedicentur-liber__latin.json',\n",
+       " 'prudentius-b-348__cathemerina__latin.json',\n",
+       " 'plautus-titus-maccius__stichus__english.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-vespasianus__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__otho__english.json',\n",
+       " 'cicero__in-defense-of-lucius-flaccus__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__clodinus-albinus__latin.json',\n",
+       " 'cicero__on-behalf-of-aulus-caecina__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__pescennius-niger__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__domitianus__latin.json',\n",
+       " 'ammianus-marcellinus__rerum-gestarum__latin.json',\n",
+       " 'ausonius-decimus-magnus__commemoratio-professorum-burdigalensium__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__ad-scapulam__latin.json',\n",
+       " 'statius-p-papinius-publius-papinius__achilleis__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-fuga-in-persecutione__latin.json',\n",
+       " 'terence__andria__english.json',\n",
+       " 'cicero__for-sextus-roscius-of-ameria__english.json',\n",
+       " 'ovid__art-of-beauty__english.json',\n",
+       " 'cicero__against-publius-servilius-rullus__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__de-anima__latin.json',\n",
+       " 'seneca-lucius-annaeus__de-otio-sapientis__latin.json',\n",
+       " 'nepos-cornelius__epaminondas__latin.json',\n",
+       " 'nepos-cornelius__cimon__latin.json',\n",
+       " 'nepos-cornelius__timoleon__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__galba__english.json',\n",
+       " 'cicero__lucullus__latin.json',\n",
+       " 'plautus-titus-maccius__poenulus__latin.json',\n",
+       " 'pliny-the-younger__epistulae__latin.json',\n",
+       " 'cicero__divinatio-against-q-caecilius__english.json',\n",
+       " 'tertullian-ca-160-ca-230__de-paenitentia__latin.json',\n",
+       " 'nepos-cornelius__miltiades__latin.json',\n",
+       " 'plautus-titus-maccius__stichus__latin.json',\n",
+       " 'jerome-saint-d-419-or-20__epistolae__latin.json',\n",
+       " 'vopiscus-flavius-fl-34-jh__goridani-tres__latin.json',\n",
+       " 'suetonius-ca-69-ca-122__divus-augustus__latin.json',\n",
+       " 'virgil__georgics__english.json',\n",
+       " 'florus-lucius-annaeus__epitome-rerum-romanorum__latin.json',\n",
+       " 'cicero__philippics__english.json',\n",
+       " 'boethius-d-524__de-consolatione-philosophiae__latin.json',\n",
+       " 'tertullian-ca-160-ca-230__adversus-judaeos-liber__latin.json',\n",
+       " 'claudianus-claudius__in-eutropium__latin.json',\n",
+       " 'cicero__in-defense-of-l-murena__english.json',\n",
+       " 'tertullian-ca-160-ca-230__adversus-praxean__latin.json']"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.listdir(perseus_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\n{\\n    \"meta\":\"section\",\\n    \"author\":\"cicero\",\\n    \"text\":{\\n        \"0\":\"\\n credo ego vos, iudices, mirari quid sit quod, cum tot summi oratores \\nhominesque nobilissimi sedeant, ego potissimum surrexerim , is qui \\nneque aetate neque ingenio neque auctoritate sim cum his qui \\nsedeant comparandus. omnes hi quos videtis adesse in hac causa \\niniuriam novo scelere conflatam putant oportere defendi, defendere \\nipsi propter iniquitatem temporum non audent. ita fit ut adsint propterea \\nquod officium sequuntur, taceant autem idcirco quia \\npericulum vitant.\",\\n        \"1\":\"\\n quid ergo? audacissimus ego ex omnibus? minime . \\n an \\n tanto officiosior quam ceteri? ne istius quidem laudis ita sum \\ncupidus ut aliis eam praereptam velim. quae me igitur res praeter \\nceteros impulit ut causam Sex. Rosci reciperem? quia , si qui istorum \\ndixisset quos videtis adesse, in quibus summa auctoritas est atque \\namplitudo, si verbum de re publica fecisset, id quod in hac causa fieri \\nnecesse est, multo plura dixisse quam dixisset putaretur.\",\\n        \"2\":\"\\n ego autem \\nsi omnia quae dicenda sunt libere dixero, nequaquam tamen similiter \\noratio mea exire atque in volgus emanare poterit. deinde quod \\nceterorum neque dictum obscurum potest esse propter nobilitatem et \\namplitudinem neque temere dicto concedi propter aetatem et\\nprudentiam. ego si quid liberius dixero, vel occultum esse propterea \\nquod nondum ad rem publicam accessi, vel ignosci adulescentiae \\nmeae poterit; tametsi non modo ignoscendi ratio verum etiam \\n cognoscendi consuetudo iam de civitate sublata est.\",\\n        \"3\":\"\\n accedit illa \\nquoque causa quod a ceteris forsitan ita petitum sit ut dicerent, ut \\nutrumvis salvo officio se facere posse arbitrarentur; a me autem ei \\ncontenderunt qui apud me et amicitia et beneficiis et dignitate \\nplurimum possunt, quorum ego nec benevolentiam erga me ignorare \\nnec auctoritatem aspernari nec voluntatem neglegere debebam . \\n\\n \\n \",\\n        \"4\":\"\\n\\n his \\nde causis ego huic causae patronus exstiti, non electus unus qui \\nmaximo ingenio sed relictus ex omnibus qui minimo periculo possem \\ndicere, neque uti satis firmo praesidio defensus Sex. Roscius verum \\nuti ne omnino desertus esset.\\n\\n\\n forsitan quaeratis qui iste terror sit et quae tanta formido quae tot \\nac talis viros impediat quo minus pro capite et fortunis alterius quem \\nad modum consuerunt causam velint dicere. quod adhuc vos \\nignorare non mirum est, propterea quod consulto ab accusatoribus \\neius rei quae conflavit hoc iudicium mentio facta non est.\",\\n        \"5\":\"\\n\\n'"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# there are many json files with the following structure:\n",
+    "\n",
+    "\"\"\"\n",
+    "{\n",
+    "    \"meta\":\"book-chapter-section\",\n",
+    "    \"author\":\"ammianus marcellinus.\",\n",
+    "    \"text\":{\n",
+    "        \"0\":{\n",
+    "            \"0\":{\n",
+    "                \"0\":\" After the survival of the events of an\\n unendurable campaign, when the spirits of both parties,\\n broken by the variety of their dangers and hardships, were still drooping,\\n before the blare of the trumpets had ceased or the soldiers been assigned to\\n their winter quarters, the gusts of raging Fortune brought new storms upon the\\n commonwealth through the misdeeds, many and notorious, of Gallus Caesar.\\n \\n He had been raised, at the very beginning of mature manhood,\\n by an unexpected promotion from the utmost depths of wretchedness to princely\\n heights, and overstepping the bounds of the authority conferred upon him, by\\n excess of violence was causing trouble everywhere. For by his relationship to\\n the imperial stock, and the affinity which he even then had with the name of\\n Constantius, he was\\n raised to such a height of presumption that, if he had been more powerful, he\\n would have ventured (it seemed) upon a course hostile to the author of his good\\n fortune. \",\n",
+    "                \"1\":\" To his cruelty his wife was besides\\n a serious incentive, a woman beyond measure presumptuous because of her kinship\\n to the emperor, and previously joined in marriage by her father Constantine\\n with his brother's son, King Hanniballianus. She, a Megaera\\n in mortal guise, constantly aroused the\\n savagery of Gallus, being as insatiable as he in her thirst for human blood.\\n The pair in process of time gradually became more expert in doing harm, and\\n through underhand and crafty eavesdroppers, who had the evil habit of lightly\\n adding to their information and wanting to learn only what was false and\\n agreeable to them, they fastened upon innocent victims false charges of\\n aspiring to royal power or of practising magic. \",\n",
+    "                \"2\":\" There stood out among their lesser atrocities, when their unbridled power\\n had already surpassed the limits of unimportant delinquencies, the sudden and\\n awful death of one Clematius, a nobleman of Alexandria. This man's\\n mother-in-law, it was said, had a violent passion for her son-in-law, but was unable to seduce him; whereupon, gaining entrance to the\\n palace by a back door, she presented the queen with a valuable necklace, and\\n thus secured the dispatch of his death-warrant to Honoratus, at that time Count\\n of the East; and so Clematius, a man contaminated by no guilt, was put to\\n death without being allowed to protest or even to open his lips.\\n \\n \",\n",
+    "                \"3\":\" After the perpetration of this impious deed,\\n which now began to arouse the fears of others also, as if cruelty were given\\n free rein, some persons were adjudged guilty on the mere shadow of suspicion\\n and condemned. Of these some were put to death, others punished by the\\n confiscation of their property and driven from their homes into exile, where,\\n having nothing left save tears and complaints, they lived on the doles of\\n charity; and since constitutional and just rule had given place to cruel\\n caprice, wealthy and famous houses were being closed. \",\n",
+    "                \"4\":\" And no words of an accuser, even though bribed, were required amid\\n these accumulations of evils, in order that these crimes might be committed, at\\n least ostensibly, under the forms of law, as has sometimes been done by cruel\\n emperors; but whatever the implacable Caesar had resolved upon was rushed to\\n fulfilment, as if it had been carefully weighed and determined to be right and\\n lawful. \",\n",
+    "                \"5\":\" It was further devised that sundry\\n low-born men, whose very insignificance made them little to be feared, should\\n be appointed to gather gossip in all quarters of Antioch and\\n report what they had heard. These, as if travellers, and in disguise, attended\\n the gatherings of distinguished citizens, and gained entrance to the houses of\\n the wealthy in the guise of needy clients; then, being secretly admitted to the\\n palace by a back door, they reported whatever they had been able to hear or\\n learn, with one accord making it a rule to add inventions of their own and make\\n doubly worse what they had learned, but suppressing the praise of Caesar which\\n the fear of impending evils extorted from some against their will. \",\n",
+    "                \"6\":\" And sometimes it happened that if the head of a\\n household, in the seclusion of his private apartments, with no confidential\\n servant present, had whispered something in the ear of his wife, the emperor\\n learned it on the following day, as if it were reported by Amphiaraus or\\n Marcius, those famous seers of old. And so even the walls, the only sharers of secrets, were\\n feared. \",\n",
+    "                \"7\":\" Moreover, his fixed purpose of\\n ferreting out these and many similar things increased, spurred on by the queen,\\n who pushed her husband's fortunes headlong to sheer ruin, when she ought\\n rather, with womanly gentleness, to have recalled him by helpful counsel to the\\n path of truth and mercy, after the manner of the wife of that savage emperor Maximinus, as we have related in our\\n account of the acts of the Gordians.\\n \\n \\n \",\n",
+    "                \"8\":\" Finally, following an unprecedented and\\n destructive course, Gallus also ventured to commit the atrocious crime which,\\n to his utter disgrace, Gallienus is said to have once hazarded at Rome. Taking\\n with him a few attendants with concealed weapons, he used to roam at evening\\n about the inns and street-corners, inquiring of every one in Greek, of which he\\n had remarkable command, what he thought of the Caesar. And this he did boldly\\n in a city where the brightness of\\n the lights at night commonly equals the resplendence of day. At last, being\\n often recognized, and reflecting that if he continued that course he would be\\n conspicuous, he appeared only in broad daylight, to attend to matters which he\\n considered important. And all this conduct of his caused very deep sorrow to\\n many.\\n \\n \",\n",
+    "                \"9\":\" Now at that time Thalassius was the\\n Praetorian Prefect at court, a man who was\\n himself of an imperious character. He, perceiving that Gallus' temper was\\n rising, to the peril of many, did not try to soothe it by ripe counsel, as\\n sometimes high officials have moderated the ire of princes; but rather roused\\n the Caesar to fury by opposing and reproving him at unseasonable times; very\\n frequently he informed the emperor of Gallus' doings, exaggerating them and\\n taking pains—whatever his motive may have been—to do it openly. Through this\\n conduct the Caesar was soon still more violently enraged, and\\n as if raising higher, as it were, the standard of his obstinacy, with no regard\\n for his own life or that of others, he rushed on with uncontrollable\\n impetuosity, like a swift torrent, to overthrow whatever opposed him.\\n \"\n",
+    "            },\n",
+    "\n",
+    "\"\"\"\n",
+    "\n",
+    "# and also, some documents don't have the same depth:\n",
+    "\n",
+    "\"\"\"\n",
+    "{\n",
+    "    \"meta\":\"section\",\n",
+    "    \"author\":\"cicero\",\n",
+    "    \"text\":{\n",
+    "        \"0\":\"\\n credo ego vos, iudices, mirari quid sit quod, cum tot summi oratores \\nhominesque nobilissimi sedeant, ego potissimum surrexerim , is qui \\nneque aetate neque ingenio neque auctoritate sim cum his qui \\nsedeant comparandus. omnes hi quos videtis adesse in hac causa \\niniuriam novo scelere conflatam putant oportere defendi, defendere \\nipsi propter iniquitatem temporum non audent. ita fit ut adsint propterea \\nquod officium sequuntur, taceant autem idcirco quia \\npericulum vitant.\",\n",
+    "        \"1\":\"\\n quid ergo? audacissimus ego ex omnibus? minime . \\n an \\n tanto officiosior quam ceteri? ne istius quidem laudis ita sum \\ncupidus ut aliis eam praereptam velim. quae me igitur res praeter \\nceteros impulit ut causam Sex. Rosci reciperem? quia , si qui istorum \\ndixisset quos videtis adesse, in quibus summa auctoritas est atque \\namplitudo, si verbum de re publica fecisset, id quod in hac causa fieri \\nnecesse est, multo plura dixisse quam dixisset putaretur.\",\n",
+    "        \"2\":\"\\n ego autem \\nsi omnia quae dicenda sunt libere dixero, nequaquam tamen similiter \\noratio mea exire atque in volgus emanare poterit. deinde quod \\nceterorum neque dictum obscurum potest esse propter nobilitatem et \\namplitudinem neque temere dicto concedi propter aetatem et\\nprudentiam. ego si quid liberius dixero, vel occultum esse propterea \\nquod nondum ad rem publicam accessi, vel ignosci adulescentiae \\nmeae poterit; tametsi non modo ignoscendi ratio verum etiam \\n cognoscendi consuetudo iam de civitate sublata est.\",\n",
+    "        \"3\":\"\\n accedit illa \\nquoque causa quod a ceteris forsitan ita petitum sit ut dicerent, ut \\nutrumvis salvo officio se facere posse arbitrarentur; a me autem ei \\ncontenderunt qui apud me et amicitia et beneficiis et dignitate \\nplurimum possunt, quorum ego nec benevolentiam erga me ignorare \\nnec auctoritatem aspernari nec voluntatem neglegere debebam . \\n\\n \\n \",\n",
+    "        \"4\":\"\\n\\n his \\nde causis ego huic causae patronus exstiti, non electus unus qui \\nmaximo ingenio sed relictus ex omnibus qui minimo periculo possem \\ndicere, neque uti satis firmo praesidio defensus Sex. Roscius verum \\nuti ne omnino desertus esset.\\n\\n\\n forsitan quaeratis qui iste terror sit et quae tanta formido quae tot \\nac talis viros impediat quo minus pro capite et fortunis alterius quem \\nad modum consuerunt causam velint dicere. quod adhuc vos \\nignorare non mirum est, propterea quod consulto ab accusatoribus \\neius rei quae conflavit hoc iudicium mentio facta non est.\",\n",
+    "        \"5\":\"\n",
+    "\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_texts_from_dict(my_dict):\n",
+    "    texts = list(my_dict.values())\n",
+    "    texts = [ re.sub(r\"\\s+\", \" \", tx) for tx in texts]\n",
+    "\n",
+    "    return texts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def recursive_get_texts_from_dict(my_dict, depth):\n",
+    "    if depth == 1:\n",
+    "        return get_texts_from_dict(my_dict)\n",
+    "    else:\n",
+    "        all_texts = []\n",
+    "        for key, val in my_dict.items():\n",
+    "            nested_texts = recursive_get_texts_from_dict(\n",
+    "                val,\n",
+    "                depth=depth-1\n",
+    "            )\n",
+    "            all_texts.extend(nested_texts)\n",
+    "        return all_texts\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def extract_text_from_perseus_dict(\n",
+    "        perseus_dict : dict,\n",
+    ")-> list[str]:\n",
+    "    perseus_text = perseus_dict[\"text\"]\n",
+    "\n",
+    "    # check the nest depth of dictionaries\n",
+    "\n",
+    "    depth = 0\n",
+    "    aux_dict = perseus_text\n",
+    "    while True:\n",
+    "        try:\n",
+    "            aux_dict = aux_dict[\"0\"]\n",
+    "            depth +=1\n",
+    "        except:\n",
+    "            break\n",
+    "\n",
+    "    return recursive_get_texts_from_dict(perseus_text, depth)\n",
+    "        \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 376/376 [00:01<00:00, 195.83it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for json_path in tqdm(os.listdir(perseus_path)):\n",
+    "    if \".json\" in json_path and \"english\" not in json_path:\n",
+    "        # load data\n",
+    "        with open(os.path.join(perseus_path, json_path), \"r\") as f:\n",
+    "            perseus_dict = json.load(f)\n",
+    "        \n",
+    "        # extract text\n",
+    "        perseus_text = extract_text_from_perseus_dict(perseus_dict)\n",
+    "\n",
+    "        \n",
+    "        \n",
+    "        # write to a txt file\n",
+    "        new_txt_path = json_path.removesuffix(\".json\")+\".txt\"\n",
+    "        new_txt_path = os.path.join(new_perseus_path, new_txt_path)\n",
+    "\n",
+    "        with open(new_txt_path, \"w\") as f:\n",
+    "            f.writelines(line + \"\\n\" for line in perseus_text)\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Processing Tesserae"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'custom_latin_corpus/tesserae'"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_tesserae_path = os.path.join(\n",
+    "    my_corpus_path, \"tesserae\"\n",
+    ")\n",
+    "if not os.path.exists(new_tesserae_path):\n",
+    "    os.mkdir(new_tesserae_path)\n",
+    "new_tesserae_path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\n<amm. 18.2.7> Quae dum diligenti maturantur effectu, Hariobaudes exploratis omnibus rediit, docuitque comperta. Post cuius adventum incitatis viribus omnes venere Mogontiacum, ubi Florentio et Lupicino (Severi successore) destinate certantibus, per pontem illic constitutum transiri debere, renitebatur firmissime Caesar, asserens pacatorum terras non debere calcari, ne (ut saepe contigit) per incivilitatem militis occurrentia vastitantis, abrupte foedera frangerentur.\\n<amm. 18.2.8> Alamanni tamen omnes quos petebat exercitus, confine periculum cogitantes, Suomarium regem amicum nobis ex pactione praeterita monuerunt minaciter, ut a transitu Romanos arceret. Eius enim pagi Rheni ripis ulterioribus adhaerebant. Quo testante resistere solum non posse, in unum coacta barbara multitude venit prope Mogontiacum, prohibitura viribus magnis exercitum, ne transmitteret flumen.\\n<amm. 18.2.9> Gemina itaque ratione visum est habile quod suaserat Caesar, ne pacatorum terrae corrumperentur, neve renitente pugnacissima plebe, pons cum multorum discrimine iungeretur iri in locum ad compaginandum pontem aptissimum.\\n'"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# files (extension .tess) are of the format:\n",
+    "\"\"\"\n",
+    "<amm. 18.2.7> Quae dum diligenti maturantur effectu, Hariobaudes exploratis omnibus rediit, docuitque comperta. Post cuius adventum incitatis viribus omnes venere Mogontiacum, ubi Florentio et Lupicino (Severi successore) destinate certantibus, per pontem illic constitutum transiri debere, renitebatur firmissime Caesar, asserens pacatorum terras non debere calcari, ne (ut saepe contigit) per incivilitatem militis occurrentia vastitantis, abrupte foedera frangerentur.\n",
+    "<amm. 18.2.8> Alamanni tamen omnes quos petebat exercitus, confine periculum cogitantes, Suomarium regem amicum nobis ex pactione praeterita monuerunt minaciter, ut a transitu Romanos arceret. Eius enim pagi Rheni ripis ulterioribus adhaerebant. Quo testante resistere solum non posse, in unum coacta barbara multitude venit prope Mogontiacum, prohibitura viribus magnis exercitum, ne transmitteret flumen.\n",
+    "<amm. 18.2.9> Gemina itaque ratione visum est habile quod suaserat Caesar, ne pacatorum terrae corrumperentur, neve renitente pugnacissima plebe, pons cum multorum discrimine iungeretur iri in locum ad compaginandum pontem aptissimum.\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 750/750 [00:02<00:00, 345.63it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "for tess_path in tqdm(os.listdir(tesserae_path)):\n",
+    "    if \".tess\" in tess_path:\n",
+    "        with open(os.path.join(tesserae_path, tess_path), \"r\") as f:\n",
+    "            text = f.readlines()\n",
+    "\n",
+    "        text = [re.sub(r\"<(.)*?>\", \"\", t) for t in text]\n",
+    "        text = [re.sub(r\"^\\s+\", \"\", t) for t in text]\n",
+    "        #text = [re.sub(r\"<\\s*[^>]*\\s*>\", \"\", t) for t in text]\n",
+    "\n",
+    "        new_txt_path = tess_path.removesuffix(\".tess\")+\".txt\"\n",
+    "        new_txt_path = os.path.join(new_tesserae_path, new_txt_path)\n",
+    "        \n",
+    "        with open(new_txt_path, \"w\") as f:\n",
+    "            f.writelines(line + \"\\n\" for line in text)\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Processing Corpus grammaticorum"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "new_corpus_grammaticorum_path = os.path.join(my_corpus_path, \"corpus_grammaticorum\")\n",
+    "if not os.path.exists(new_corpus_grammaticorum_path):\n",
+    "    os.mkdir(new_corpus_grammaticorum_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\n{\\n   \"author\":\"Alcuinus\",\\n   \"title\":\"Alcuinus de orthographia\",\\n   \"edition_new\":\"S. Bruni 1997\",\\n   \"edition_keil\":\"GL 7,295-312\",\\n   \"text\":\" me legat antiquas cupiat qui scire loquelas / me spernens loquitur mox sine lege patrum. | <littera a> aeternus, aetas, aeuum per duo u, aequitas, aequus id est iustus, haec omnia per ae diptongon scribenda sunt; equus, si animal significat per \\n\\\\nsimplicem e. |accusso per duo c et per duo s scribi debet, accedo per duo c. saepe ad, euphoniae causa, in sequentem mutabitur consonantem, ut afficio \\n\\\\naffluo a\\n\\n'"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# JSON files with the following format\n",
+    "\"\"\"\n",
+    "{\n",
+    "   \"author\":\"Alcuinus\",\n",
+    "   \"title\":\"Alcuinus de orthographia\",\n",
+    "   \"edition_new\":\"S. Bruni 1997\",\n",
+    "   \"edition_keil\":\"GL 7,295-312\",\n",
+    "   \"text\":\" me legat antiquas cupiat qui scire loquelas / me spernens loquitur mox sine lege patrum. | <littera a> aeternus, aetas, aeuum per duo u, aequitas, aequus id est iustus, haec omnia per ae diptongon scribenda sunt; equus, si animal significat per \n",
+    "\\\\nsimplicem e. |accusso per duo c et per duo s scribi debet, accedo per duo c. saepe ad, euphoniae causa, in sequentem mutabitur consonantem, ut afficio \n",
+    "\\\\naffluo a\n",
+    "\n",
+    "\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# there are problems with the double backslash\n",
+    "class LazyDecoder(json.JSONDecoder):\n",
+    "    def decode(self, s, **kwargs):\n",
+    "        regex_replacements = [\n",
+    "            (re.compile(r\"\\\\\\\\n\"), r\"\\n\" )\n",
+    "            #(re.compile(r'([^\\\\])\\\\([^\\\\])'), r'\\1\\\\\\\\\\2'),\n",
+    "            #(re.compile(r',(\\s*])'), r'\\1'),\n",
+    "        ]\n",
+    "        for regex, replacement in regex_replacements:\n",
+    "            s = regex.sub(replacement, s)\n",
+    "        return super().decode(s, **kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 44/44 [00:00<00:00, 78.65it/s] \n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 40,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "errors = 0\n",
+    "for autor_path in tqdm(os.listdir(corpus_grammaticorum_path)):\n",
+    "    sub_dir = os.path.join(corpus_grammaticorum_path, autor_path)\n",
+    "\n",
+    "\n",
+    "\n",
+    "    if not os.path.isdir(sub_dir):\n",
+    "        continue\n",
+    "    for json_path in os.listdir(\n",
+    "            sub_dir\n",
+    "            ):\n",
+    "        if \".json\" in json_path:\n",
+    "            full_path = os.path.join(\n",
+    "                corpus_grammaticorum_path,\n",
+    "                autor_path,\n",
+    "                json_path\n",
+    "            )\n",
+    "            with open(full_path) as f:\n",
+    "                #content_dict = json.load(f, cls=LazyDecoder)\n",
+    "                content_raw = f.read()\n",
+    "            content_raw = re.sub(r\"\\\\\\\\n\", r\"\", content_raw)\n",
+    "            content_raw = re.sub(r\"\\\\.\", r\"\", content_raw)\n",
+    "            # content_raw = re.sub(r\"\\\\n\", r\"\", content_raw)\n",
+    "            content_raw = re.sub(r\"\\|\", r\" \", content_raw)\n",
+    "            content_raw = re.sub(r\"/\", r\" \", content_raw)\n",
+    "            content_raw = re.sub(r\"\\s+\", r\" \", content_raw)\n",
+    "            content_raw = re.sub(r\"[<>\\[\\]\\(\\)»«]\", r\"\", content_raw)\n",
+    "\n",
+    "            # deleting some separators\n",
+    "            content_raw = re.sub(r\"\\|\\d+\\|\", r\"\", content_raw)\n",
+    "\n",
+    "            # deleting page numbers\n",
+    "            content_raw = re.sub(r\"\\d+\", r\"\", content_raw)\n",
+    "\n",
+    "            # delete some extra quotation marks\n",
+    "            replacement = (\n",
+    "                lambda match: (\n",
+    "                    '\"text\": \"'\n",
+    "                    + match.group(1).replace('\"', '')\n",
+    "                    + '\"\\n}'\n",
+    "                    )\n",
+    "            )\n",
+    "\n",
+    "            content_raw = re.sub(\n",
+    "                r'\"text\": \"(.*?)\"\\s}',\n",
+    "                replacement,\n",
+    "                content_raw\n",
+    "            )\n",
+    "\n",
+    "            try:\n",
+    "                content_dict = json.loads(content_raw)\n",
+    "            except Exception as e:\n",
+    "                print(str(e))\n",
+    "                print(json_path)\n",
+    "                print(full_path)\n",
+    "                print(content_raw)\n",
+    "                errors +=1\n",
+    "                continue\n",
+    "                \n",
+    "\n",
+    "            text = content_dict[\"text\"]\n",
+    "\n",
+    "            new_txt_path = json_path.removesuffix(\".json\")+\".txt\"\n",
+    "            new_txt_path = os.path.join(new_corpus_grammaticorum_path, new_txt_path)\n",
+    "\n",
+    "            with open(new_txt_path, \"w\",encoding=\"utf-8\") as f:\n",
+    "                f.write(text)\n",
+    "\n",
+    "errors"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "44"
+      ]
+     },
+     "execution_count": 39,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(os.listdir(corpus_grammaticorum_path))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bertenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

training_notebooks/02_more_preprocessing_and_training_WP_tokenizer.ipynb

@@ -0,0 +1,210 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Now training Wordpiece with instructions from huggingface\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# https://colab.research.google.com/github/huggingface/notebooks/blob/master/examples/tokenizer_training.ipynb\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# join all txt files into a single txt file\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import time\n",
+    "\n",
+    "paths = [str(x) for x in Path(\"./custom_latin_corpus\").glob(\"**/*.txt\")]\n",
+    "all_text = []\n",
+    "for path in paths:\n",
+    "    with open(path, \"r\") as f:\n",
+    "        text = f.read()\n",
+    "\n",
+    "    all_text.append(text)\n",
+    "# text batch size\n",
+    "batch_size = 100\n",
+    "def batch_iterator():\n",
+    "    for i in range(0, len(all_text), batch_size):\n",
+    "        yield all_text[i : i + batch_size]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"03_full_latin_corpus_for_training.txt\", \"w\") as f:\n",
+    "    f.writelines(all_text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "from tokenizers import decoders, models, normalizers, pre_tokenizers, processors, trainers, Tokenizer\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tokenizer = Tokenizer(models.WordPiece(unk_token=\"[UNK]\"))\n",
+    "tokenizer.normalizer = normalizers.BertNormalizer(lowercase=True)\n",
+    "tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()\n",
+    "tokenizer.pre_tokenizer.pre_tokenize_str(\"This is an example!\")\n",
+    "special_tokens = [\"[UNK]\", \"[PAD]\", \"[CLS]\", \"[SEP]\", \"[MASK]\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "trainer = trainers.WordPieceTrainer(\n",
+    "    vocab_size=25000, \n",
+    "    special_tokens=special_tokens,\n",
+    "    min_frequency=2,\n",
+    "    limit_alphabet=50\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "\n",
+    "\n",
+    "\n",
+    "tokenizer.train_from_iterator(batch_iterator(), trainer=trainer)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2 3\n"
+     ]
+    }
+   ],
+   "source": [
+    "# now, define the post processor\n",
+    "cls_token_id = tokenizer.token_to_id(\"[CLS]\")\n",
+    "sep_token_id = tokenizer.token_to_id(\"[SEP]\")\n",
+    "print(cls_token_id, sep_token_id)\n",
+    "tokenizer.post_processor = processors.TemplateProcessing(\n",
+    "    single=f\"[CLS]:0 $A:0 [SEP]:0\",\n",
+    "    pair=f\"[CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1\",\n",
+    "    special_tokens=[\n",
+    "        (\"[CLS]\", cls_token_id),\n",
+    "        (\"[SEP]\", sep_token_id),\n",
+    "    ],\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test an example\n",
+    "example_encoding = tokenizer.encode(\"Roma in Italia est.\", \"Italia in Europa est.\")\n",
+    "example_encoding.tokens\n",
+    "tokenizer.decoder = decoders.WordPiece(prefix=\"##\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('latin_WP_tokenizer/tokenizer_config.json',\n",
+       " 'latin_WP_tokenizer/special_tokens_map.json',\n",
+       " 'latin_WP_tokenizer/vocab.txt',\n",
+       " 'latin_WP_tokenizer/added_tokens.json',\n",
+       " 'latin_WP_tokenizer/tokenizer.json')"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# wrap it inside transformers object\n",
+    "\n",
+    "from transformers import BertTokenizerFast\n",
+    "\n",
+    "new_wp_tokenizer = BertTokenizerFast(tokenizer_object=tokenizer)\n",
+    "new_wp_tokenizer.save_pretrained(\n",
+    "    \"latin_WP_tokenizer\"\n",
+    ")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bertenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

training_notebooks/04_train_mlm_script.py

@@ -0,0 +1,90 @@
+
+
+# https://colab.research.google.com/github/huggingface/blog/blob/main/notebooks/01_how_to_train.ipynb#scrollTo=VNZZs-r6iKAV
+
+from transformers import Trainer, TrainingArguments
+from transformers import DataCollatorForLanguageModeling
+from transformers import LineByLineTextDataset
+from transformers import BertForMaskedLM
+from transformers import BertTokenizerFast, BertTokenizer
+from transformers import BertConfig
+import torch
+import time
+print(torch.cuda.is_available())
+
+# Start the timer
+start_time = time.time()
+# Define a config
+
+
+config = BertConfig(
+    vocab_size=25000,
+    max_position_embeddings=512,
+    num_attention_heads=12,
+    num_hidden_layers=12,
+    type_vocab_size=2,
+)
+
+
+tokenizer = BertTokenizerFast.from_pretrained(
+    "latin_WP_tokenizer",
+)
+
+# initialize from config to train from scratch
+
+
+model = BertForMaskedLM(config=config)
+
+print(f"There are {model.num_parameters()} parameters")
+
+
+full_corpus_file = "03_full_latin_corpus_for_training.txt"
+
+
+dataset = LineByLineTextDataset(
+    tokenizer=tokenizer,
+    file_path=full_corpus_file,
+    block_size=128,
+)
+
+
+data_collator = DataCollatorForLanguageModeling(
+    tokenizer=tokenizer, mlm=True, mlm_probability=0.15
+)
+
+# initialize trainer
+
+
+output_dir = "./Latin_BERT_training_2"
+
+training_args = TrainingArguments(
+    output_dir=output_dir,
+    overwrite_output_dir=True,
+    # num_train_epochs=3,  # like the original BERT
+    num_train_epochs=1,  # just one epoch
+    per_device_train_batch_size=64,
+    save_steps=10000,
+    save_total_limit=2,
+    prediction_loss_only=True,
+)
+
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    data_collator=data_collator,
+    train_dataset=dataset,
+)
+
+
+# now train
+trainer.train()
+
+trainer.save_model("./latin_BERT_2")
+
+
+# End the timer
+end_time = time.time()
+# Calculate the elapsed time
+elapsed_time = end_time - start_time
+# Print the elapsed time
+print(f"Elapsed time: {elapsed_time} seconds")

training_notebooks/05_loading_and_saving_pipeline.ipynb

@@ -0,0 +1,57 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fill_mask = pipeline(\n",
+    "    \"fill-mask\",\n",
+    "    model=\"latin_BERT_2\",\n",
+    "    tokenizer=\"latin_WP_tokenizer\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fill_mask.save_pretrained(\"latin_BERT_final\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dhvenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

training_notebooks/05_testing_latin_bert.ipynb

@@ -0,0 +1,187 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "fill_mask = pipeline(\n",
+    "    \"fill-mask\",\n",
+    "    model=\"latin_BERT_final\",\n",
+    "    tokenizer=\"latin_WP_tokenizer\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "False False False False False\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.05949964374303818,\n",
+       "  'token': 18,\n",
+       "  'token_str': '.',\n",
+       "  'sequence': 'roma in. est.'},\n",
+       " {'score': 0.05125246196985245,\n",
+       "  'token': 16,\n",
+       "  'token_str': ',',\n",
+       "  'sequence': 'roma in, est.'},\n",
+       " {'score': 0.014972569420933723,\n",
+       "  'token': 870,\n",
+       "  'token_str': 'et',\n",
+       "  'sequence': 'roma in et est.'},\n",
+       " {'score': 0.009671307168900967,\n",
+       "  'token': 879,\n",
+       "  'token_str': '##que',\n",
+       "  'sequence': 'roma inque est.'},\n",
+       " {'score': 0.007990601472556591,\n",
+       "  'token': 30,\n",
+       "  'token_str': ':',\n",
+       "  'sequence': 'roma in : est.'}]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fill_mask(\"Roma in [MASK] est.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "False False False False False\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.05949964374303818,\n",
+       "  'token': 18,\n",
+       "  'token_str': '.',\n",
+       "  'sequence': 'ubi est.?.'},\n",
+       " {'score': 0.05125246196985245,\n",
+       "  'token': 16,\n",
+       "  'token_str': ',',\n",
+       "  'sequence': 'ubi est,?.'},\n",
+       " {'score': 0.014972569420933723,\n",
+       "  'token': 870,\n",
+       "  'token_str': 'et',\n",
+       "  'sequence': 'ubi est et?.'},\n",
+       " {'score': 0.009671302512288094,\n",
+       "  'token': 879,\n",
+       "  'token_str': '##que',\n",
+       "  'sequence': 'ubi estque?.'},\n",
+       " {'score': 0.007990599609911442,\n",
+       "  'token': 30,\n",
+       "  'token_str': ':',\n",
+       "  'sequence': 'ubi est :?.'}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fill_mask(\"Ubi est [MASK] ?.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "False False False False False\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.05949964374303818,\n",
+       "  'token': 18,\n",
+       "  'token_str': '.',\n",
+       "  'sequence': 'de honoratorum.'},\n",
+       " {'score': 0.05125246196985245,\n",
+       "  'token': 16,\n",
+       "  'token_str': ',',\n",
+       "  'sequence': 'de honoratorum,'},\n",
+       " {'score': 0.014972569420933723,\n",
+       "  'token': 870,\n",
+       "  'token_str': 'et',\n",
+       "  'sequence': 'de honoratorum et'},\n",
+       " {'score': 0.009671302512288094,\n",
+       "  'token': 879,\n",
+       "  'token_str': '##que',\n",
+       "  'sequence': 'de honoratorumque'},\n",
+       " {'score': 0.007990601472556591,\n",
+       "  'token': 30,\n",
+       "  'token_str': ':',\n",
+       "  'sequence': 'de honoratorum :'}]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# De honoratorum vehiculis\n",
+    "fill_mask(\"De honoratorum [MASK]\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bertenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

training_notebooks/06_testing_latin_bert.ipynb

@@ -0,0 +1,244 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "fill_mask = pipeline(\n",
+    "    \"fill-mask\",\n",
+    "    \"latin_BERT_final\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.010115200653672218,\n",
+       "  'token': 265,\n",
+       "  'token_str': 'hoc',\n",
+       "  'sequence': 'roma in hoc est.'},\n",
+       " {'score': 0.004335678182542324,\n",
+       "  'token': 1156,\n",
+       "  'token_str': 'deo',\n",
+       "  'sequence': 'roma in deo est.'},\n",
+       " {'score': 0.003660168731585145,\n",
+       "  'token': 146,\n",
+       "  'token_str': 'non',\n",
+       "  'sequence': 'roma in non est.'},\n",
+       " {'score': 0.0034285704605281353,\n",
+       "  'token': 745,\n",
+       "  'token_str': 'nobis',\n",
+       "  'sequence': 'roma in nobis est.'},\n",
+       " {'score': 0.0032894855830818415,\n",
+       "  'token': 971,\n",
+       "  'token_str': 'rebus',\n",
+       "  'sequence': 'roma in rebus est.'}]"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fill_mask(\"Roma in [MASK] est.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.04990068078041077,\n",
+       "  'token': 145,\n",
+       "  'token_str': 'est',\n",
+       "  'sequence': 'ubi est est?.'},\n",
+       " {'score': 0.01739734411239624,\n",
+       "  'token': 215,\n",
+       "  'token_str': 'quid',\n",
+       "  'sequence': 'ubi est quid?.'},\n",
+       " {'score': 0.008733403868973255,\n",
+       "  'token': 391,\n",
+       "  'token_str': 'mihi',\n",
+       "  'sequence': 'ubi est mihi?.'},\n",
+       " {'score': 0.007146364543586969,\n",
+       "  'token': 368,\n",
+       "  'token_str': 'sum',\n",
+       "  'sequence': 'ubi est sum?.'},\n",
+       " {'score': 0.006486538797616959,\n",
+       "  'token': 425,\n",
+       "  'token_str': 'tibi',\n",
+       "  'sequence': 'ubi est tibi?.'}]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fill_mask(\"Ubi est [MASK] ?.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.7615262269973755,\n",
+       "  'token': 12,\n",
+       "  'token_str': '.',\n",
+       "  'sequence': 'de honoratorum.'},\n",
+       " {'score': 0.03385818004608154,\n",
+       "  'token': 23,\n",
+       "  'token_str': ':',\n",
+       "  'sequence': 'de honoratorum :'},\n",
+       " {'score': 0.02129465527832508,\n",
+       "  'token': 10,\n",
+       "  'token_str': ',',\n",
+       "  'sequence': 'de honoratorum,'},\n",
+       " {'score': 0.014383483678102493,\n",
+       "  'token': 25,\n",
+       "  'token_str': '?',\n",
+       "  'sequence': 'de honoratorum?'},\n",
+       " {'score': 0.008870471268892288,\n",
+       "  'token': 109,\n",
+       "  'token_str': 'et',\n",
+       "  'sequence': 'de honoratorum et'}]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# De honoratorum vehiculis\n",
+    "fill_mask(\"De honoratorum [MASK]\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.00837777741253376,\n",
+       "  'token': 1838,\n",
+       "  'token_str': 'urbe',\n",
+       "  'sequence': 'gallia est omnis divisa in urbe tres'},\n",
+       " {'score': 0.007593569345772266,\n",
+       "  'token': 1628,\n",
+       "  'token_str': 'corpore',\n",
+       "  'sequence': 'gallia est omnis divisa in corpore tres'},\n",
+       " {'score': 0.007336211856454611,\n",
+       "  'token': 2035,\n",
+       "  'token_str': 'medio',\n",
+       "  'sequence': 'gallia est omnis divisa in medio tres'},\n",
+       " {'score': 0.006218622904270887,\n",
+       "  'token': 983,\n",
+       "  'token_str': 'parte',\n",
+       "  'sequence': 'gallia est omnis divisa in parte tres'},\n",
+       " {'score': 0.0054352362640202045,\n",
+       "  'token': 238,\n",
+       "  'token_str': 'quo',\n",
+       "  'sequence': 'gallia est omnis divisa in quo tres'}]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fill_mask(\"Gallia est omnis divisa in [MASK] tres\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'score': 0.008044794201850891,\n",
+       "  'token': 1628,\n",
+       "  'token_str': 'corpore',\n",
+       "  'sequence': 'gallia est omnis divisa in corpore tres.'},\n",
+       " {'score': 0.00732386251911521,\n",
+       "  'token': 1838,\n",
+       "  'token_str': 'urbe',\n",
+       "  'sequence': 'gallia est omnis divisa in urbe tres.'},\n",
+       " {'score': 0.0072334203869104385,\n",
+       "  'token': 983,\n",
+       "  'token_str': 'parte',\n",
+       "  'sequence': 'gallia est omnis divisa in parte tres.'},\n",
+       " {'score': 0.006316048558801413,\n",
+       "  'token': 2035,\n",
+       "  'token_str': 'medio',\n",
+       "  'sequence': 'gallia est omnis divisa in medio tres.'},\n",
+       " {'score': 0.004988126456737518,\n",
+       "  'token': 1177,\n",
+       "  'token_str': 'terra',\n",
+       "  'sequence': 'gallia est omnis divisa in terra tres.'}]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Adding a point changes the predictions!\n",
+    "fill_mask(\"Gallia est omnis divisa in [MASK] tres.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bertenv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}