new mapudungun word list

app.py

@@ -4,14 +4,19 @@ import re
 import random
 
 ### load and prepare corpus
-corpus = util.load_raw_text(corpus_directory="map_avenue")
+#corpus = util.load_raw_text(corpus_directory="map_avenue")
 
-corpus = corpus.lower()
-word_regex = r"[a-z]+"
-def tokenize(text: str):
-    return re.findall(word_regex, text)
+corpus = util.load_single_raw_text_file("mapudungun.easy.filtered")
+
+#corpus = corpus.lower()
+#word_regex = r"[a-z]+"
+#def tokenize(text: str):
+#    return re.findall(word_regex, text)
+
+#words = tokenize(corpus)
+words = corpus.split()
+print(words)
 
-words = tokenize(corpus)
 
 lexicon = set()
 for word in words:

mapudungun.easy.filtered

@@ -0,0 +1,71 @@
+diwlliñ
+chod
+kompañ
+ñukekantu
+kallfü
+chaw
+epu
+mellfü
+küla
+kawellu
+wün'
+üñüm
+kinchika
+namün'
+kiñe
+kelüchod
+mari
+kal'
+pütra
+yu
+lig
+ufisa
+pangi
+ñuke
+challwa
+chawkantu
+karü
+kadü
+milla
+kurü
+küwü
+kolü
+kelü
+waka
+dewü
+ñarki
+kapüra
+llampüdken
+llalliñ
+fallke
+kechu
+pilun
+ligkelü
+aylla
+pura
+paine
+nawel
+awawe
+pel'
+mañke
+trulitruli
+longko
+wala
+trewa
+achawall
+meli
+chang
+regle
+kayu
+foro
+lien
+filu
+kewün'
+peyu
+rukü
+luku
+furi
+lipang
+koneku
+nge
+sañwe