src/prepare_offline.py

def noop_load(*args, **kwargs):
    return None


def go_prepare_offline(*args, **kwargs):
    kwargs0 = kwargs['kwargs']
    # gen.py steps should have already obtained:
    #   model+tokenizers from base_model or model_lock if required
    #   tokenizers, including tokenizers for model_lock if using inference servers even if no LLM locally
    #   score_model or reward model
    #
    # Additional steps are related to document Q/A:
    # For simplicity use gradio functions,
    #  but not API calls that would require actual gradio app up and API usage that might have issues

    kwargs['max_quality'] = True
    embed = True
    h2ogpt_key = ''
    file_list = ['tests/driverslicense.jpeg', 'tests/CityofTshwaneWater.pdf', 'tests/example.xlsx']

    inputs2 = [kwargs['my_db_state0'],
               kwargs['selection_docs_state0'],
               kwargs['requests_state0'],
               kwargs0['langchain_mode'],
               kwargs0['chunk'],
               kwargs0['chunk_size'],
               embed,
               kwargs['image_audio_loaders_options'],
               kwargs['pdf_loaders_options'],
               kwargs['url_loaders_options'],
               kwargs['jq_schema0'],
               kwargs['extract_frames'],
               h2ogpt_key,
               ]

    for fileup_output in file_list:
        inputs1 = [fileup_output]
        add_file_kwargs = dict(fn=kwargs['update_db_func'],
                               inputs=inputs1 + inputs2)
        add_file_kwargs['fn'](*tuple(add_file_kwargs['inputs']))

        # ensure normal blip (not 2) obtained
        blip2 = 'CaptionBlip2'
        if blip2 in kwargs['image_audio_loaders_options']:
            image_audio_loaders_options = kwargs['image_audio_loaders_options'].copy()
            image_audio_loaders_options.remove(blip2)

        # ensure normal asr (not asrlarge) obtained
        asrlarge = 'ASRLarge'
        if asrlarge in kwargs['image_audio_loaders_options']:
            image_audio_loaders_options = kwargs['image_audio_loaders_options'].copy()
            image_audio_loaders_options.remove(asrlarge)

        inputs2[8] = kwargs['image_audio_loaders_options']
        add_file_kwargs = dict(fn=kwargs['update_db_func'],
                               inputs=inputs1 + inputs2)
        add_file_kwargs['fn'](*tuple(add_file_kwargs['inputs']))

    # FakeTokenizer etc. needs tiktoken for general tasks
    import tiktoken
    encoding = tiktoken.get_encoding("cl100k_base")
    assert encoding
    encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")
    assert encoding

    # sometimes summarization needs gpt2 still
    from transformers import AutoTokenizer
    model_name = 'gpt2'
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    assert tokenizer

    # then run h2ogpt as:
    # HF_DATASETS_OFFLINE=1 TRANSFORMERS_OFFLINE=1 python generate.py --gradio_offline_level=2 --share=False ...