Upload folder using huggingface_hub

config.json

@@ -63,10 +63,10 @@
         "max_input_len": 1024,
         "max_seq_len": 1048576,
         "opt_batch_size": null,
-        "max_batch_size": 256,
+        "max_batch_size": 128,
         "max_beam_width": 1,
-        "max_num_tokens": 4096,
-        "opt_num_tokens": null,
+        "max_num_tokens": 32000,
+        "opt_num_tokens": 128,
         "max_prompt_embedding_table_size": 0,
         "kv_cache_type": "PAGED",
         "gather_context_logits": false,
@@ -122,8 +122,8 @@
         "plugin_config": {
             "dtype": "float16",
             "bert_attention_plugin": "auto",
-            "gpt_attention_plugin": "float16",
-            "gemm_plugin": "float16",
+            "gpt_attention_plugin": "auto",
+            "gemm_plugin": "auto",
             "gemm_swiglu_plugin": null,
             "fp8_rowwise_gemm_plugin": null,
             "smooth_quant_gemm_plugin": null,
@@ -149,7 +149,7 @@
             "tokens_per_block": 64,
             "use_paged_context_fmha": true,
             "use_fp8_context_fmha": false,
-            "multiple_profiles": true,
+            "multiple_profiles": false,
             "paged_state": false,
             "streamingllm": false,
             "manage_weights": false,

rank0.engine

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1cff19f5c52a4e565b8e011c091863a93a8f31dc16f9c738711770b932b4a283
-size 16171359220
+oid sha256:9ab2e652421c6f2f5afb545f7d710aa08783d94a93eb5911af856d9b1da6bd3c
+size 16135219868