| { | |
| "base_current_gpu_type": "NVIDIA A100-PCIE-40GB", | |
| "base_current_gpu_total_memory": 40339.3125, | |
| "base_token_generation_latency_sync": 49.01026916503906, | |
| "base_token_generation_latency_async": 49.090784043073654, | |
| "base_token_generation_throughput_sync": 0.020403887124809734, | |
| "base_token_generation_throughput_async": 0.020370422259350582, | |
| "base_token_generation_CO2_emissions": null, | |
| "base_token_generation_energy_consumption": null, | |
| "base_inference_latency_sync": 47.847014617919925, | |
| "base_inference_latency_async": 46.59438133239746, | |
| "base_inference_throughput_sync": 0.020899945544888283, | |
| "base_inference_throughput_async": 0.021461815167501574, | |
| "base_inference_CO2_emissions": null, | |
| "base_inference_energy_consumption": null, | |
| "smashed_current_gpu_type": "NVIDIA A100-PCIE-40GB", | |
| "smashed_current_gpu_total_memory": 40339.3125, | |
| "smashed_token_generation_latency_sync": 186.75454864501953, | |
| "smashed_token_generation_latency_async": 186.877416446805, | |
| "smashed_token_generation_throughput_sync": 0.005354621920887112, | |
| "smashed_token_generation_throughput_async": 0.005351101374438424, | |
| "smashed_token_generation_CO2_emissions": null, | |
| "smashed_token_generation_energy_consumption": null, | |
| "smashed_inference_latency_sync": 193.4655502319336, | |
| "smashed_inference_latency_async": 165.618634223938, | |
| "smashed_inference_throughput_sync": 0.005168878897566845, | |
| "smashed_inference_throughput_async": 0.006037967917594766, | |
| "smashed_inference_CO2_emissions": null, | |
| "smashed_inference_energy_consumption": null | |
| } |