philschmid
/

meta-llama-3.1-8b-instruct-trt-l4

Inference Endpoints

Model card Files Files and versions Community

philschmid HF staff commited on 21 days ago

Commit

0694451

•

1 Parent(s): ed667f7

Upload folder using huggingface_hub

Files changed (2) hide show

engines/config.json +2 -2
engines/rank0.engine +2 -2

engines/config.json CHANGED Viewed

@@ -63,9 +63,9 @@
         "max_input_len": 8000,
         "max_seq_len": 8192,
         "opt_batch_size": 8,
-        "max_batch_size": 32,
         "max_beam_width": 1,
-        "max_num_tokens": 262144,
         "opt_num_tokens": null,
         "max_prompt_embedding_table_size": 0,
         "kv_cache_type": "PAGED",

         "max_input_len": 8000,
         "max_seq_len": 8192,
         "opt_batch_size": 8,
+        "max_batch_size": 8,
         "max_beam_width": 1,
+        "max_num_tokens": 65536,
         "opt_num_tokens": null,
         "max_prompt_embedding_table_size": 0,
         "kv_cache_type": "PAGED",

engines/rank0.engine CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9237017b5eba0f364725a5d1d67fd215793151dd4720acdd9e33dfd73aca9b1e
-size 16155886596

 version https://git-lfs.github.com/spec/v1
+oid sha256:7df1fe24805ba9c3f1f650025f616f9df0e9455181784e81aab11de59aceed1c
+size 16155880388