{ | |
"princeton-nlp/Sheared-LLaMA-1.3B": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"defog/sqlcoder-7b-2": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"m-a-p/OpenCodeInterpreter-DS-6.7B": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"ibm/labradorite-13b": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 8, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 8, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 8, | |
"sequence_length": 4096, | |
"num_cores": 8, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"gorilla-llm/gorilla-openfunctions-v2": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"m-a-p/ChatMusician": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"LargeWorldModel/LWM-Text-Chat-1M": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 2, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"01-ai/Yi-34B-200K": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 24, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 24, | |
"auto_cast_type": "fp16" | |
} | |
], | |
"abacusai/Smaug-72B-v0.1": [ | |
{ | |
"batch_size": 1, | |
"sequence_length": 4096, | |
"num_cores": 24, | |
"auto_cast_type": "fp16" | |
}, | |
{ | |
"batch_size": 4, | |
"sequence_length": 4096, | |
"num_cores": 24, | |
"auto_cast_type": "fp16" | |
} | |
] | |
} | |