feat: update parse_ipa to fit new lexicon format

ipa/ipa.py

@@ -63,26 +63,21 @@ def get_ipa(raw_text, dialect):
     
     return final_words, final_ipa, final_pinyin, missing_words
 
-def parse_ipa(ipa: str):
+def parse_ipa(ipa: str, delete_chars="\+\-\|\_", as_space=""):
     text = []
-    ipa_list = re.split(r"(?<![， -])(?=[， -])|(?<=[， -])(?![， -])",ipa)
-    # tone as a separate token
-    for phoneme_with_tone in ipa_list:
-        if phoneme_with_tone ==" ":
-            text.append(phoneme_with_tone)
-            continue
-        elif phoneme_with_tone == "，":
-            text.extend(" ， ")
-            continue
-        elif phoneme_with_tone == "-": # use " " split 詞 (or use " " to split 字)
-            continue
-
-        split_phoneme_and_tone = phoneme_with_tone.split("_")
-
-        if len(split_phoneme_and_tone) == 2:
-            phoneme, tone = split_phoneme_and_tone
-            text.extend(phoneme) 
-            text.append(tone)
+    
+    ipa_list = re.split(r"(?<![\d])(?=[\d])|(?<=[\d])(?![\d])", ipa)
+    print(ipa_list)
+    for word in ipa_list:
+        if word.isdigit():
+            text.append(word)
         else:
-            text.extend(split_phoneme_and_tone[0])
+            if len(as_space) > 0:
+                word = re.sub(r"[{}]".format(as_space), " ", word)
+            if len(delete_chars) > 0:
+                word = re.sub(r"[{}]".format(delete_chars), "", word)
+                
+            word = word.replace("，", " ， ")
+            text.extend(word)
+
     return text