updated with paper link
Browse files- PartialDD.png +0 -0
- app.py +27 -27
PartialDD.png
CHANGED
app.py
CHANGED
@@ -59,11 +59,11 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
59 |
|
60 |
gr.Markdown(
|
61 |
"""
|
62 |
-
<img src='https://huggingface.co/spaces/bkhmsi/Partial-Arabic-Diacritization/resolve/main/PartialDD.png' style='float:right; margin: 0 0 10px 10px;'/>
|
63 |
|
64 |
<h1> Partial Diacritization: A Context-Contrastive Inference Approach </h1>
|
65 |
<h2> Authors: Muhammad ElNokrashy, Badr AlKhamissi </h2>
|
66 |
-
<h3> Paper
|
67 |
<details>
|
68 |
<summary>Abstract</summary>
|
69 |
<p>Diacritization plays a pivotal role in improving readability and disambiguating the meaning of Arabic texts. Efforts have so far focused on marking every eligible character (Full Diacritization). Comparatively overlooked, Partial Diacritzation (PD) is the selection of a subset of characters to be marked to aid comprehension where needed.Research has indicated that excessive diacritic marks can hinder skilled readers---reducing reading speed and accuracy. We conduct a behavioral experiment and show that partially marked text is often easier to read than fully marked text, and sometimes easier than plain text. In this light, we introduce Context-Contrastive Partial Diacritization (CCPD)---a novel approach to PD which integrates seamlessly with existing Arabic diacritization systems. CCPD processes each word twice, once with context and once without, and diacritizes only the characters with disparities between the two inferences. Further, we introduce novel indicators for measuring partial diacritization quality {SR, PDER, HDER, ERE}, essential for establishing this as a machine learning task. Lastly, we introduce TD2, a Transformer-variant of an established model which offers a markedly different performance profile on our proposed indicators compared to all other known systems.</p>
|
@@ -77,9 +77,12 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
77 |
value=current_model_name
|
78 |
)
|
79 |
|
80 |
-
with gr.Tab(label="
|
81 |
-
|
82 |
-
|
|
|
|
|
|
|
83 |
placeholder="ุงูุชุจ ููุง",
|
84 |
lines=5,
|
85 |
label="Input",
|
@@ -88,7 +91,7 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
88 |
text_align='right',
|
89 |
)
|
90 |
|
91 |
-
|
92 |
lines=5,
|
93 |
label="Output",
|
94 |
type='text',
|
@@ -97,25 +100,24 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
97 |
show_copy_button=True,
|
98 |
)
|
99 |
|
100 |
-
|
101 |
-
|
102 |
|
103 |
gr.Examples(
|
104 |
examples=[
|
105 |
-
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "TD2"],
|
|
|
|
|
106 |
],
|
107 |
-
inputs=[
|
108 |
-
outputs=
|
109 |
-
fn=
|
110 |
cache_examples=True,
|
111 |
)
|
112 |
|
113 |
-
with gr.Tab(label="
|
114 |
-
|
115 |
-
|
116 |
-
threshold_slider = gr.Slider(label="Soft Masking Threshold", minimum=0, maximum=1, value=0.1)
|
117 |
-
|
118 |
-
partial_input_txt = gr.Textbox(
|
119 |
placeholder="ุงูุชุจ ููุง",
|
120 |
lines=5,
|
121 |
label="Input",
|
@@ -124,7 +126,7 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
124 |
text_align='right',
|
125 |
)
|
126 |
|
127 |
-
|
128 |
lines=5,
|
129 |
label="Output",
|
130 |
type='text',
|
@@ -133,18 +135,16 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
|
|
133 |
show_copy_button=True,
|
134 |
)
|
135 |
|
136 |
-
|
137 |
-
|
138 |
|
139 |
gr.Examples(
|
140 |
examples=[
|
141 |
-
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "
|
142 |
-
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "Soft", 0.1, "TD2"],
|
143 |
-
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "Soft", 0.01, "TD2"],
|
144 |
],
|
145 |
-
inputs=[
|
146 |
-
outputs=
|
147 |
-
fn=
|
148 |
cache_examples=True,
|
149 |
)
|
150 |
|
|
|
59 |
|
60 |
gr.Markdown(
|
61 |
"""
|
62 |
+
<img src='https://huggingface.co/spaces/bkhmsi/Partial-Arabic-Diacritization/resolve/main/PartialDD.png' style='float:right; margin: 0 0 10px 10px; width: 20%'/>
|
63 |
|
64 |
<h1> Partial Diacritization: A Context-Contrastive Inference Approach </h1>
|
65 |
<h2> Authors: Muhammad ElNokrashy, Badr AlKhamissi </h2>
|
66 |
+
<h3> Paper: <a href="https://arxiv.org/abs/2401.08919"> https://arxiv.org/abs/2401.08919 </a> </h3>
|
67 |
<details>
|
68 |
<summary>Abstract</summary>
|
69 |
<p>Diacritization plays a pivotal role in improving readability and disambiguating the meaning of Arabic texts. Efforts have so far focused on marking every eligible character (Full Diacritization). Comparatively overlooked, Partial Diacritzation (PD) is the selection of a subset of characters to be marked to aid comprehension where needed.Research has indicated that excessive diacritic marks can hinder skilled readers---reducing reading speed and accuracy. We conduct a behavioral experiment and show that partially marked text is often easier to read than fully marked text, and sometimes easier than plain text. In this light, we introduce Context-Contrastive Partial Diacritization (CCPD)---a novel approach to PD which integrates seamlessly with existing Arabic diacritization systems. CCPD processes each word twice, once with context and once without, and diacritizes only the characters with disparities between the two inferences. Further, we introduce novel indicators for measuring partial diacritization quality {SR, PDER, HDER, ERE}, essential for establishing this as a machine learning task. Lastly, we introduce TD2, a Transformer-variant of an established model which offers a markedly different performance profile on our proposed indicators compared to all other known systems.</p>
|
|
|
77 |
value=current_model_name
|
78 |
)
|
79 |
|
80 |
+
with gr.Tab(label="Partial Diacritization") as partial_settings:
|
81 |
+
with gr.Row():
|
82 |
+
masking_mode = gr.Radio(choices=["Hard", "Soft"], value="Hard", label="Masking Mode")
|
83 |
+
threshold_slider = gr.Slider(label="Soft Masking Threshold", minimum=0, maximum=1, value=0.1)
|
84 |
+
|
85 |
+
partial_input_txt = gr.Textbox(
|
86 |
placeholder="ุงูุชุจ ููุง",
|
87 |
lines=5,
|
88 |
label="Input",
|
|
|
91 |
text_align='right',
|
92 |
)
|
93 |
|
94 |
+
partial_output_txt = gr.Textbox(
|
95 |
lines=5,
|
96 |
label="Output",
|
97 |
type='text',
|
|
|
100 |
show_copy_button=True,
|
101 |
)
|
102 |
|
103 |
+
partial_btn = gr.Button(value="Shakkel")
|
104 |
+
partial_btn.click(diacritze_partial, inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice], outputs=[partial_output_txt], queue=True)
|
105 |
|
106 |
gr.Examples(
|
107 |
examples=[
|
108 |
+
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "Hard", 0, "TD2"],
|
109 |
+
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "Soft", 0.1, "TD2"],
|
110 |
+
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "Soft", 0.01, "TD2"],
|
111 |
],
|
112 |
+
inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice],
|
113 |
+
outputs=partial_output_txt,
|
114 |
+
fn=diacritze_partial,
|
115 |
cache_examples=True,
|
116 |
)
|
117 |
|
118 |
+
with gr.Tab(label="Full Diacritization"):
|
119 |
+
|
120 |
+
full_input_txt = gr.Textbox(
|
|
|
|
|
|
|
121 |
placeholder="ุงูุชุจ ููุง",
|
122 |
lines=5,
|
123 |
label="Input",
|
|
|
126 |
text_align='right',
|
127 |
)
|
128 |
|
129 |
+
full_output_txt = gr.Textbox(
|
130 |
lines=5,
|
131 |
label="Output",
|
132 |
type='text',
|
|
|
135 |
show_copy_button=True,
|
136 |
)
|
137 |
|
138 |
+
full_btn = gr.Button(value="Shakkel")
|
139 |
+
full_btn.click(diacritze_full, inputs=[full_input_txt, model_choice], outputs=[full_output_txt], queue=True)
|
140 |
|
141 |
gr.Examples(
|
142 |
examples=[
|
143 |
+
["ููู ุญู
ู ู
ู ู
ุฌูุณ ุงูุฎูุงุฑ ุ ููู
ูู
ูุน ู
ู ุงูููุงู
", "TD2"],
|
|
|
|
|
144 |
],
|
145 |
+
inputs=[full_input_txt, model_choice],
|
146 |
+
outputs=full_output_txt,
|
147 |
+
fn=diacritze_full,
|
148 |
cache_examples=True,
|
149 |
)
|
150 |
|