Add tokenizer_config.json download

app.py

@@ -45,6 +45,12 @@ if 'uuid' not in st.session_state:
 if 'successful_template' not in st.session_state:
     st.session_state['successful_template'] = ''
 
+if 'generated_prompt_w_add_generation_prompt' not in st.session_state:
+    st.session_state['generated_prompt_w_add_generation_prompt'] = ''
+    
+if 'generated_prompt_wo_add_generation_prompt' not in st.session_state:
+    st.session_state['generated_prompt_wo_add_generation_prompt'] = ''
+
 if not os.path.exists("./tmp"):
     os.makedirs("./tmp")
 
@@ -97,7 +103,7 @@ if st.session_state['tokenizer_json'] is not None:
         {"role": "user", "content": "I'd like to show off how chat templating works!"},
         ]
         st.markdown("### Example Conversation")
-        st.json(json.dumps(chat, indent=4))
+        st.json(json.dumps(chat, indent=4), expanded=False)
 
         prompt_template_col, prompt_template_output_col = st.columns(spec=[0.3, 0.7])
 
@@ -141,22 +147,35 @@ if st.session_state['tokenizer_json'] is not None:
                 generated_prompt_wo_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= False)
                 generated_prompt_w_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= True)
 
+                st.session_state['successful_template'] = copy.deepcopy(st.session_state['input_jinja_template'])
+
+
+            if len(st.session_state['successful_template']) > 0:
                 st.text_area(
-                    "Generate Prompt with `add_generation_prompt=False`", value=generated_prompt_wo_add_generation_prompt, 
+                    "Generate Prompt with `add_generation_prompt=False`", value=st.session_state['generated_prompt_wo_add_generation_prompt'], 
                     height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_wo_add_generation_prompt")
 
                 st.text_area(
-                    "Generate Prompt with `add_generation_prompt=True`", value=generated_prompt_w_add_generation_prompt, 
+                    "Generate Prompt with `add_generation_prompt=True`", value=st.session_state['generated_prompt_w_add_generation_prompt'], 
                     height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_w_add_generation_prompt")
-
-                st.session_state['successful_template'] = copy.deepcopy(st.session_state['input_jinja_template'])
-
-            if len(st.session_state['successful_template']) > 0:
+                    
                 access_token_no_cache = st.text_input("HuggingFace Access Token API with Write Access", type="password", key="access_token_no_cache")
                 commit_message_text_input = st.text_input("Commit Message", key="commit_message_text_input")
                 to_private_checkbox = st.checkbox("To Private Repo", key="to_private_checkbox")
-                push_to_hub_button = st.button("Push to Hub", key="push_to_hub_button")
                 create_pr_checkbox = st.checkbox("Create PR (For Contribution 🤗)", key="create_pr_checkbox")
+                push_to_hub_button = st.button("Push to Hub", key="push_to_hub_button", use_container_width=True)
+                st.session_state['tokenizer'].save_pretrained(f"./tmp/{st.session_state['uuid']}_{hf_model_repo_name}")
+                with open(f"./tmp/{st.session_state['uuid']}_{hf_model_repo_name}/tokenizer_config.json", "r") as f:
+
+                    tokenizer_config_content = json.loads(f.read())
+
+                st.download_button(
+                    label="Download tokenizer_config.json",
+                    data=json.dumps(tokenizer_config_content, indent=4),
+                    file_name='tokenizer_config.json',
+                    mime='application/json',
+                    use_container_width=True
+                )
                 if push_to_hub_button:
                     with open(f"./tmp/{st.session_state['uuid']}/tmp_chat_template.json", "w") as fp:
                         fp.write(st.session_state['successful_template'])