Added token autocompletion lookup

When edit/adding token_id metadata you will now get an extra token lookup to quickly find the correct token.

app.py

@@ -106,12 +106,11 @@ with gr.Blocks(
     )
 
     with gr.Row():
-        # Too slow unfortunately, needs a proper search box
-        # meta_lookup = gr.Dropdown(
-        #     label = 'Lookup token',
-        #     type = 'index',
-        #     visible = False,
-        # )
+        meta_lookup = gr.Dropdown(
+            label = 'Lookup token',
+            allow_custom_value = True,
+            visible = False,
+        )
 
         meta_number = gr.Number(
             visible = False,
@@ -364,8 +363,11 @@ with gr.Blocks(
         typ = None
         if (val := meta.key.get(key, standard_metadata.get(key))) is not None:
             typ = GGUFValueType(val[0]).name
-        elif key and key.startswith('tokenizer.chat_template.'):
-            typ = GGUFValueType.STRING.name
+        elif key:
+            if key.startswith('tokenizer.chat_template.'):
+                typ = GGUFValueType.STRING.name
+            elif key.endswith('_token_id'):
+                typ = GGUFValueType.UINT32.name
 
         return {
             meta_types: gr.Dropdown(
@@ -391,6 +393,7 @@ with gr.Blocks(
         ],
         outputs = [
             meta_boolean,
+            meta_lookup,
             meta_number,
             meta_string,
             meta_array,
@@ -402,6 +405,8 @@ with gr.Blocks(
         typ: int,
     ):
         val = None
+        tokens = meta.key.get('tokenizer.ggml.tokens', (-1, []))[1]
+
         if (data := meta.key.get(key, standard_metadata.get(key))) is not None:
             typ = data[0]
             val = data[1]
@@ -423,6 +428,11 @@ with gr.Blocks(
                 value = val if typ == GGUFValueType.BOOL and data is not None else False,
                 visible = True if typ == GGUFValueType.BOOL else False,
             ),
+            meta_lookup: gr.Dropdown(
+                None,
+                value = tokens[val] if is_number and data is not None and key.endswith('_token_id') and val < len(tokens) else '',
+                visible = True if is_number and key.endswith('_token_id') else False,
+            ),
             meta_number: gr.Number(
                 value = val if is_number and data is not None else 0,
                 precision = 10 if typ == GGUFValueType.FLOAT32 or typ == GGUFValueType.FLOAT64 else 0,
@@ -537,6 +547,36 @@ with gr.Blocks(
         )
 
 
+    @gr.on(
+        triggers = [
+            meta_lookup.key_up,
+        ],
+        inputs = [
+            meta_state,
+        ],
+        outputs = [
+            meta_lookup,
+        ],
+        show_progress = 'hidden',
+        trigger_mode = 'always_last',
+    )
+    def token_lookup(
+        meta: MetadataState,
+        keyup: gr.KeyUpData,
+    ):
+        found = []
+        value = keyup.input_value.lower()
+        tokens = meta.key.get('tokenizer.ggml.tokens', (-1, []))[1]
+
+        any(((found.append(t), len(found) > 5)[1] for i, t in enumerate(tokens) if value in t.lower()))
+
+        return {
+            meta_lookup: gr.Dropdown(
+                found,
+            ),
+        }
+
+
     def add_metadata(
         meta: MetadataState,
         key: str,
@@ -550,7 +590,7 @@ with gr.Blocks(
 
             return {
                 meta_changes: gr.HighlightedText(
-                )
+                ),
             }
 
         if key in meta.rem:
@@ -570,6 +610,45 @@ with gr.Blocks(
         )
 
 
+    def token_to_id(
+        meta: MetadataState,
+        token: str,
+    ):
+        tokens = meta.key.get('tokenizer.ggml.tokens', (-1, []))[1]
+
+        try:
+            found = tokens.index(token)
+        except Exception as e:
+            raise gr.Error('Token not found')
+
+        return {
+            meta_number: gr.Number(
+                found,
+            ),
+        }
+
+
+    meta_lookup.input(
+        token_to_id,
+        inputs = [
+            meta_state,
+            meta_lookup,
+        ],
+        outputs = [
+            meta_number,
+        ],
+    ).success(
+        add_metadata,
+        inputs = [
+            meta_state,
+            meta_keys,
+            meta_types,
+            meta_number,
+        ],
+        outputs = [
+        ] + state_change_components,
+    )
+
     meta_boolean.input(
         add_metadata,
         inputs = [
@@ -582,12 +661,6 @@ with gr.Blocks(
         ] + state_change_components,
     )
 
-    # meta_lookup.input(
-    #     lambda token: gr.Number(value = token),
-    #     inputs = meta_lookup,
-    #     outputs = meta_number,
-    # )
-
     meta_number.submit(
         add_metadata,
         inputs = [