Update with h2oGPT hash f121dcf534b7c7da96e22fdfb00a7436503f167e

app.py

@@ -2,10 +2,11 @@ import functools
 import inspect
 import sys
 import os
+import time
 import traceback
 import typing
-
-from utils import set_seed, flatten_list, clear_torch_cache, system_info_print
+import filelock
+from utils import set_seed, flatten_list, clear_torch_cache, system_info_print, zip_data, save_generate_output
 
 SEED = 1236
 set_seed(SEED)
@@ -27,10 +28,11 @@ from finetune import get_loaders, example_data_points, generate_prompt, get_gith
     human, bot, prompt_type_to_model_name, inv_prompt_type_to_model_lower
 from stopping import CallbackToGenerator, Stream, StoppingCriteriaSub
 
-is_hf = os.getenv("HUGGINGFACE_SPACES")
-is_gpth2oai = os.getenv("GPT_H2O_AI")
+is_hf = bool(os.getenv("HUGGINGFACE_SPACES"))
+is_gpth2oai = bool(os.getenv("GPT_H2O_AI"))
 is_public = is_hf or is_gpth2oai  # multi-user case with fixed model and disclaimer
 is_low_mem = is_hf  # assumes run on 24GB consumer GPU
+admin_pass = os.getenv("ADMIN_PASS")
 
 
 def main(
@@ -58,6 +60,7 @@ def main(
 
         llama_type: bool = None,
         debug: bool = False,
+        save_path: str = None,
         share: bool = True,
         local_files_only: bool = False,
         resume_download: bool = True,
@@ -111,6 +114,7 @@ def main(
     if is_hf:
         # must override share if in spaces
         share = False
+    save_path = os.getenv('SAVE_PATH')
 
     # get defaults
     model_lower = base_model.lower()
@@ -178,7 +182,7 @@ def main(
             if not eval_sharegpt_as_output:
                 model, tokenizer, device = get_model(**locals())
                 model_state = [model, tokenizer, device, base_model]
-                fun = partial(evaluate, model_state, debug=debug, chat=chat)
+                fun = partial(evaluate, model_state, debug=debug, chat=chat, save_path=save_path)
             else:
                 assert eval_sharegpt_prompts_only > 0
 
@@ -542,8 +546,9 @@ def go_gradio(**kwargs):
     if is_public:
         description += """<p><b> DISCLAIMERS: </b><ul><i><li>The data used to train this model include The Pile and other sources. These may contain objectionable content, so the model may reproduce that material. Use application and responses at own risk.</i></li>"""
         if kwargs['load_8bit']:
-            description += """<i><li> Model is loaded in 8-bit and with other limitations in order to fit on GPUs with lower amounts of VRAM, so UX can be worse than non-hosted version.</i></li>"""
-        description += """<i><li>Model loading and unloading disabled to avoid GPU OOM for multi-user environment.</i></li></ul></p>"""
+            description += """<i><li> Model is loaded in 8-bit, model loading-unloading is disabled, and other limitations exist in order to fit on GPUs with lower amounts of VRAM, so UX can be worse than non-hosted version.</i></li>"""
+        description += """<i><li>Conversations may be used to improve h2oGPT.  Do not share sensitive information.</i></li>"""
+        description += """<i><li>By using h2oGPT, you accept our [Terms of Service](https://github.com/h2oai/h2ogpt/blob/main/tos.md).</i></li></ul></p>"""
 
     if kwargs['verbose']:
         task_info_md = f"""
@@ -551,14 +556,43 @@ def go_gradio(**kwargs):
     else:
         task_info_md = ''
 
-    css_code = """footer {visibility: hidden}
-body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en/site/header/master/_jcr_content/root/container/header_copy/logo.coreimg.svg/1678976605175/h2o-logo.svg");}}"""
+    css_code = """footer {visibility: hidden;}
+body{background:linear-gradient(#f5f5f5,#e5e5e5);}
+body.dark{background:linear-gradient(#0d0d0d,#333333);}"""
 
-    from gradio.themes.utils import colors, fonts, sizes
+    from gradio.themes.utils import Color, colors, fonts, sizes
     if kwargs['h2ocolors']:
-        colors_dict = dict(primary_hue=colors.yellow,
-                           secondary_hue=colors.yellow,
-                           neutral_hue=colors.gray,
+        h2o_yellow = Color(
+            name="yellow",
+            c50="#fffef2",
+            c100="#fff9e6",
+            c200="#ffecb3",
+            c300="#ffe28c",
+            c400="#ffd659",
+            c500="#fec925",
+            c600="#e6ac00",
+            c700="#bf8f00",
+            c800="#a67c00",
+            c900="#664d00",
+            c950="#403000",
+        )
+        h2o_gray = Color(
+            name="gray",
+            c50="#f2f2f2",
+            c100="#e5e5e5",
+            c200="#cccccc",
+            c300="#b2b2b2",
+            c400="#999999",
+            c500="#7f7f7f",
+            c600="#666666",
+            c700="#4c4c4c",
+            c800="#333333",
+            c900="#191919",
+            c950="#0d0d0d",
+        )
+        colors_dict = dict(primary_hue=h2o_yellow,
+                           secondary_hue=h2o_yellow,
+                           neutral_hue=h2o_gray,
                            spacing_size=sizes.spacing_md,
                            radius_size=sizes.radius_md,
                            text_size=sizes.text_md,
@@ -635,7 +669,7 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
 
         # go button visible if
         base_wanted = bool(kwargs['base_model']) and kwargs['login_mode_if_model0']
-        go_btn = gr.Button(value="LOGIN", visible=base_wanted, variant="primary")
+        go_btn = gr.Button(value="ENTER", visible=base_wanted, variant="primary")
         normal_block = gr.Row(visible=not base_wanted)
         with normal_block:
             with gr.Tabs():
@@ -770,12 +804,27 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
                                     add_model_button = gr.Button("Add new model name")
                                     add_lora_button = gr.Button("Add new LORA name", visible=kwargs['show_lora'])
                 with gr.TabItem("System"):
-                    with gr.Row():
+                    system_row = gr.Row(visible=not is_public)
+                    admin_pass_textbox = gr.Textbox(label="Admin Password", type='password', visible=is_public)
+                    admin_btn = gr.Button(value="admin", visible=is_public)
+                    with system_row:
                         with gr.Column():
                             system_text = gr.Textbox(label='System Info')
                             system_btn = gr.Button(value='Get System Info')
 
+                            zip_btn = gr.Button("Zip")
+                            file_output = gr.File()
 
+        # Get flagged data
+        zip_data1 = functools.partial(zip_data, root_dirs=['flagged_data_points', kwargs['save_path']])
+        zip_btn.click(zip_data1, inputs=None, outputs=file_output)
+
+        def check_admin_pass(x):
+            return gr.update(visible=x == admin_pass)
+
+        admin_btn.click(check_admin_pass, inputs=admin_pass_textbox, outputs=system_row)
+
+        # Get inputs to evaluate()
         inputs_list = get_inputs_list(locals(), kwargs['model_lower'])
         from functools import partial
         all_kwargs = kwargs.copy()
@@ -1094,7 +1143,7 @@ body{background-image:url("https://h2o.ai/content/experience-fragments/h2o/us/en
 
 
 input_args_list = ['model_state']
-inputs_kwargs_list = ['debug', 'chat', 'hard_stop_list', 'sanitize_bot_response', 'model_state0']
+inputs_kwargs_list = ['debug', 'chat', 'save_path', 'hard_stop_list', 'sanitize_bot_response', 'model_state0']
 
 
 def get_inputs_list(inputs_dict, model_lower):
@@ -1157,6 +1206,7 @@ def evaluate(
         src_lang=None,
         tgt_lang=None,
         debug=False,
+        save_path=None,
         chat=False,
         hard_stop_list=None,
         sanitize_bot_response=True,
@@ -1369,6 +1419,8 @@ def evaluate(
                     raise StopIteration
                 yield prompter.get_response(decoded_output, prompt=inputs_decoded,
                                             sanitize_bot_response=sanitize_bot_response)
+            if save_path:
+                save_generate_output(output=decoded_output, base_model=base_model, json_file_path=save_path)
             return
         else:
             outputs = model.generate(**gen_kwargs)
@@ -1585,5 +1637,9 @@ if __name__ == "__main__":
 
     python generate.py --base_model='togethercomputer/GPT-NeoXT-Chat-Base-20B' --prompt_type='human_bot' --lora_weights='GPT-NeoXT-Chat-Base-20B.merged.json.8_epochs.57b2892c53df5b8cefac45f84d019cace803ef26.28'
 
+    must have 4*48GB GPU and run without 8bit in order for sharding to work with infer_devices=False
+    can also pass --prompt_type='human_bot' and model can somewhat handle instructions without being instruct tuned
+    python generate.py --base_model=decapoda-research/llama-65b-hf --load_8bit=False --infer_devices=False --prompt_type='human_bot'
+
     """, flush=True)
     fire.Fire(main)

finetune.py

@@ -73,6 +73,7 @@ prompt_type_to_model_name = {
         'decapoda-research/llama-7b-hf',
         'decapoda-research/llama-13b-hf',
         'decapoda-research/llama-30b-hf',
+        'decapoda-research/llama-65b-hf',
         'facebook/mbart-large-50-many-to-many-mmt',
         'philschmid/bart-large-cnn-samsum',
         'philschmid/flan-t5-base-samsum',

utils.py

@@ -1,7 +1,13 @@
+import contextlib
 import os
 import gc
 import random
+import shutil
 import time
+import traceback
+import zipfile
+
+import filelock
 import numpy as np
 import pandas as pd
 import torch
@@ -87,3 +93,61 @@ def system_info_print():
         return df.to_markdown()
     except Exception as e:
         return "Error: %s" % str(e)
+
+
+def zip_data(root_dirs=None, zip_path='data.zip', base_dir='./'):
+    try:
+        return _zip_data(zip_path=zip_path, base_dir=base_dir, root_dirs=root_dirs)
+    except Exception as e:
+        traceback.print_exc()
+        print('Exception in zipping: %s' % str(e))
+
+
+def _zip_data(root_dirs=None, zip_path='data.zip', base_dir='./'):
+    assert root_dirs is not None
+    with zipfile.ZipFile(zip_path, "w") as expt_zip:
+        for root_dir in root_dirs:
+            if root_dir is None:
+                continue
+            for root, d, files in os.walk(root_dir):
+                for file in files:
+                    file_to_archive = os.path.join(root, file)
+                    assert os.path.exists(file_to_archive)
+                    path_to_archive = os.path.relpath(file_to_archive, base_dir)
+                    expt_zip.write(filename=file_to_archive, arcname=path_to_archive)
+    return "data.zip"
+
+
+def save_generate_output(output=None, base_model=None, json_file_path=None):
+    try:
+        return _save_generate_output(output=output, base_model=base_model, json_file_path=json_file_path)
+    except Exception as e:
+        traceback.print_exc()
+        print('Exception in saving: %s' % str(e))
+
+
+def _save_generate_output(output=None, base_model=None, json_file_path=None):
+    """
+    Save conversation to .json, row by row
+    Appends if file exists
+    """
+    assert isinstance(json_file_path, str), "must provide save_path"
+    as_file = os.path.normpath(json_file_path)
+    if os.path.isfile(as_file):
+        # protection if had file there before
+        os.remove(as_file)
+    os.makedirs(json_file_path, exist_ok=True)
+    json_file_file = os.path.join(json_file_path, 'save.json')
+    import json
+    if output[-10:] == '\n\n<human>:':
+        # remove trailing <human>:
+        output = output[:-10]
+    with filelock.FileLock("save_path.lock"):
+        # lock logging in case have concurrency
+        with open(json_file_file, "a") as f:
+            # just add [ at start, and ] at end, and have proper JSON dataset
+            f.write(
+                "  " + json.dumps(
+                    dict(text=output, time=time.ctime(), base_model=base_model)
+                ) + ",\n"
+            )