| { | |
| "schemaVersion": 1, | |
| "deviceProperties": [ | |
| { | |
| "id": 0, "name": "NVIDIA A100-PCIE-40GB", "totalGlobalMem": 42411098112, | |
| "computeMajor": 8, "computeMinor": 0, | |
| "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048, | |
| "regsPerBlock": 65536, "regsPerMultiprocessor": 65536, "warpSize": 32, | |
| "sharedMemPerBlock": 49152, "sharedMemPerMultiprocessor": 167936, | |
| "numSms": 108, "sharedMemPerBlockOptin": 166912 | |
| } | |
| ], | |
| "with_modules": 1, | |
| "record_shapes": 1, | |
| "with_stack": 1, | |
| "profile_memory": 1, | |
| "traceEvents": [ | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276743, "dur": 383, | |
| "args": { | |
| "Python parent id": null, "Python id": 1, "Ev Idx": 0 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276744, "dur": 382, | |
| "args": { | |
| "Python parent id": 1, "Python id": 2, "Ev Idx": 1 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276746, "dur": 380, | |
| "args": { | |
| "Python parent id": 2, "Python id": 3, "Ev Idx": 2 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(323): check_internal_messages", "pid": 0, "tid": 0, | |
| "ts": 1733485661276746, "dur": 380, | |
| "args": { | |
| "Python parent id": 3, "Python id": 4, "Ev Idx": 3 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733485661276748, "dur": 378, | |
| "args": { | |
| "Python parent id": 4, "Python id": 5, "Ev Idx": 4 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276749, "dur": 377, | |
| "args": { | |
| "Python parent id": 5, "Python id": 6, "Ev Idx": 5 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276751, "dur": 375, | |
| "args": { | |
| "Python parent id": 6, "Python id": 7, "Ev Idx": 6 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276753, "dur": 373, | |
| "args": { | |
| "Python parent id": 7, "Python id": 8, "Ev Idx": 7 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276753, "dur": 373, | |
| "args": { | |
| "Python parent id": 8, "Python id": 9, "Ev Idx": 8 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276754, "dur": 372, | |
| "args": { | |
| "Python parent id": 9, "Python id": 10, "Ev Idx": 9 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(279): check_network_status", "pid": 0, "tid": 0, | |
| "ts": 1733485661276754, "dur": 372, | |
| "args": { | |
| "Python parent id": 10, "Python id": 11, "Ev Idx": 10 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733485661276755, "dur": 371, | |
| "args": { | |
| "Python parent id": 11, "Python id": 12, "Ev Idx": 11 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276756, "dur": 370, | |
| "args": { | |
| "Python parent id": 12, "Python id": 13, "Ev Idx": 12 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276756, "dur": 370, | |
| "args": { | |
| "Python parent id": 13, "Python id": 14, "Ev Idx": 13 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276762, "dur": 364, | |
| "args": { | |
| "Python parent id": 14, "Python id": 15, "Ev Idx": 14 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276762, "dur": 364, | |
| "args": { | |
| "Python parent id": 15, "Python id": 16, "Ev Idx": 15 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276763, "dur": 363, | |
| "args": { | |
| "Python parent id": 16, "Python id": 17, "Ev Idx": 16 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(303): check_stop_status", "pid": 0, "tid": 0, | |
| "ts": 1733485661276764, "dur": 362, | |
| "args": { | |
| "Python parent id": 17, "Python id": 18, "Ev Idx": 17 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0, | |
| "ts": 1733485661276764, "dur": 362, | |
| "args": { | |
| "Python parent id": 18, "Python id": 19, "Ev Idx": 18 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276765, "dur": 361, | |
| "args": { | |
| "Python parent id": 19, "Python id": 20, "Ev Idx": 19 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276765, "dur": 361, | |
| "args": { | |
| "Python parent id": 20, "Python id": 21, "Ev Idx": 20 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276770, "dur": 356, | |
| "args": { | |
| "Python parent id": 21, "Python id": 22, "Ev Idx": 21 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276770, "dur": 356, | |
| "args": { | |
| "Python parent id": 22, "Python id": 23, "Ev Idx": 22 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276771, "dur": 355, | |
| "args": { | |
| "Python parent id": 23, "Python id": 24, "Ev Idx": 23 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router.py(70): message_loop", "pid": 0, "tid": 0, | |
| "ts": 1733485661276771, "dur": 355, | |
| "args": { | |
| "Python parent id": 24, "Python id": 25, "Ev Idx": 24 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router_sock.py(27): _read_message", "pid": 0, "tid": 0, | |
| "ts": 1733485661276772, "dur": 354, | |
| "args": { | |
| "Python parent id": 25, "Python id": 26, "Ev Idx": 25 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(285): read_server_response", "pid": 0, "tid": 0, | |
| "ts": 1733485661276773, "dur": 353, | |
| "args": { | |
| "Python parent id": 26, "Python id": 27, "Ev Idx": 26 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(255): _read_packet_bytes", "pid": 0, "tid": 0, | |
| "ts": 1733485661276774, "dur": 352, | |
| "args": { | |
| "Python parent id": 27, "Python id": 28, "Ev Idx": 27 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276779, "dur": 347, | |
| "args": { | |
| "Python parent id": 28, "Python id": 29, "Ev Idx": 28 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276779, "dur": 347, | |
| "args": { | |
| "Python parent id": 29, "Python id": 30, "Ev Idx": 29 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276780, "dur": 346, | |
| "args": { | |
| "Python parent id": 30, "Python id": 31, "Ev Idx": 30 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276780, "dur": 346, | |
| "args": { | |
| "Python parent id": 31, "Python id": 32, "Ev Idx": 31 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276781, "dur": 345, | |
| "args": { | |
| "Python parent id": 32, "Python id": 33, "Ev Idx": 32 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0, | |
| "ts": 1733485661276784, "dur": 342, | |
| "args": { | |
| "Python parent id": 33, "Python id": 34, "Ev Idx": 33 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0, | |
| "ts": 1733485661276784, "dur": 342, | |
| "args": { | |
| "Python parent id": 34, "Python id": 35, "Ev Idx": 34 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0, | |
| "ts": 1733485661276784, "dur": 342, | |
| "args": { | |
| "Python parent id": 35, "Python id": 36, "Ev Idx": 35 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276785, "dur": 341, | |
| "args": { | |
| "Python parent id": 36, "Python id": 37, "Ev Idx": 36 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0, | |
| "ts": 1733485661276785, "dur": 341, | |
| "args": { | |
| "Python parent id": 37, "Python id": 38, "Ev Idx": 37 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "ft_llm.py(298): <module>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276789, "dur": 337, | |
| "args": { | |
| "Python parent id": null, "Python id": 39, "Ev Idx": 38 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "ft_llm.py(260): train", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276790, "dur": 336, | |
| "args": { | |
| "Python parent id": 39, "Python id": 40, "Ev Idx": 39 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(699): __exit__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276792, "dur": 334, | |
| "args": { | |
| "Python parent id": 40, "Python id": 41, "Ev Idx": 40 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(715): stop", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276793, "dur": 333, | |
| "args": { | |
| "Python parent id": 41, "Python id": 42, "Ev Idx": 41 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(744): _transit_action", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276793, "dur": 333, | |
| "args": { | |
| "Python parent id": 42, "Python id": 43, "Ev Idx": 42 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(161): start_trace", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276794, "dur": 197, | |
| "args": { | |
| "Python parent id": 43, "Python id": 44, "Ev Idx": 43 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(289): _start_trace", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276795, "dur": 74, | |
| "args": { | |
| "Python parent id": 44, "Python id": 45, "Ev Idx": 44 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276875, "dur": 27, | |
| "args": { | |
| "Python parent id": 44, "Python id": 46, "Ev Idx": 45 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276896, "dur": 5, | |
| "args": { | |
| "Python parent id": 46, "Python id": 47, "Ev Idx": 46 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276904, "dur": 2, | |
| "args": { | |
| "Python parent id": 44, "Python id": 48, "Ev Idx": 47 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276904, "dur": 2, | |
| "args": { | |
| "Python parent id": 48, "Python id": 49, "Ev Idx": 48 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276907, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 50, "Ev Idx": 49 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276907, "dur": 0, | |
| "args": { | |
| "Python parent id": 50, "Python id": 51, "Ev Idx": 50 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276908, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 52, "Ev Idx": 51 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276908, "dur": 0, | |
| "args": { | |
| "Python parent id": 52, "Python id": 53, "Ev Idx": 52 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method kineto_available of PyCapsule object at 0x7f99f72d3390>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276911, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 54, "Ev Idx": 53 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(279): _get_distributed_info", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276913, "dur": 41, | |
| "args": { | |
| "Python parent id": 44, "Python id": 55, "Ev Idx": 54 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/__init__.py(9): is_available", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276921, "dur": 23, | |
| "args": { | |
| "Python parent id": 55, "Python id": 56, "Ev Idx": 55 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276943, "dur": 1, | |
| "args": { | |
| "Python parent id": 56, "Python id": 57, "Ev Idx": 56 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(973): is_initialized", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276947, "dur": 7, | |
| "args": { | |
| "Python parent id": 55, "Python id": 58, "Ev Idx": 57 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(588): WORLD", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276949, "dur": 4, | |
| "args": { | |
| "Python parent id": 58, "Python id": 59, "Ev Idx": 58 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(460): default_pg", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276953, "dur": 0, | |
| "args": { | |
| "Python parent id": 59, "Python id": 60, "Ev Idx": 59 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276955, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 61, "Ev Idx": 60 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(356): __getattr__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276978, "dur": 8, | |
| "args": { | |
| "Python parent id": 44, "Python id": 62, "Ev Idx": 61 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(140): __getattr__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276984, "dur": 2, | |
| "args": { | |
| "Python parent id": 62, "Python id": 63, "Ev Idx": 62 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method items of dict object at 0x7f9995dba9c0>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276990, "dur": 0, | |
| "args": { | |
| "Python parent id": 44, "Python id": 64, "Ev Idx": 63 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(195): stop_trace", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276993, "dur": 133, | |
| "args": { | |
| "Python parent id": 43, "Python id": 65, "Ev Idx": 64 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(291): __exit__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276995, "dur": 131, | |
| "args": { | |
| "Python parent id": 65, "Python id": 66, "Ev Idx": 65 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(782): synchronize", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661276999, "dur": 127, | |
| "args": { | |
| "Python parent id": 66, "Python id": 67, "Ev Idx": 66 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277001, "dur": 6, | |
| "args": { | |
| "Python parent id": 67, "Python id": 68, "Ev Idx": 67 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277003, "dur": 4, | |
| "args": { | |
| "Python parent id": 68, "Python id": 69, "Ev Idx": 68 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277005, "dur": 2, | |
| "args": { | |
| "Python parent id": 69, "Python id": 70, "Ev Idx": 69 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(360): __init__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277011, "dur": 72, | |
| "args": { | |
| "Python parent id": 67, "Python id": 71, "Ev Idx": 70 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/_utils.py(9): _get_device_index", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277013, "dur": 68, | |
| "args": { | |
| "Python parent id": 71, "Python id": 72, "Ev Idx": 71 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277014, "dur": 1, | |
| "args": { | |
| "Python parent id": 72, "Python id": 73, "Ev Idx": 72 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277015, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 74, "Ev Idx": 73 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277016, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 75, "Ev Idx": 74 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277019, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 76, "Ev Idx": 75 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277020, "dur": 0, | |
| "args": { | |
| "Python parent id": 72, "Python id": 77, "Ev Idx": 76 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(759): _get_device_index", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277026, "dur": 55, | |
| "args": { | |
| "Python parent id": 72, "Python id": 78, "Ev Idx": 77 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277027, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 79, "Ev Idx": 78 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277028, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 80, "Ev Idx": 79 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277028, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 81, "Ev Idx": 80 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277029, "dur": 0, | |
| "args": { | |
| "Python parent id": 78, "Python id": 82, "Ev Idx": 81 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(733): _get_current_device_index", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277030, "dur": 51, | |
| "args": { | |
| "Python parent id": 78, "Python id": 83, "Ev Idx": 82 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(721): _get_device_attr", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277032, "dur": 48, | |
| "args": { | |
| "Python parent id": 83, "Python id": 84, "Ev Idx": 83 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(708): _get_available_device_type", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277033, "dur": 25, | |
| "args": { | |
| "Python parent id": 84, "Python id": 85, "Ev Idx": 84 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(105): is_available", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277035, "dur": 23, | |
| "args": { | |
| "Python parent id": 85, "Python id": 86, "Ev Idx": 85 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(96): _is_compiled", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277038, "dur": 2, | |
| "args": { | |
| "Python parent id": 86, "Python id": 87, "Ev Idx": 86 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277040, "dur": 0, | |
| "args": { | |
| "Python parent id": 87, "Python id": 88, "Ev Idx": 87 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(101): _nvml_based_avail", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277041, "dur": 13, | |
| "args": { | |
| "Python parent id": 86, "Python id": 89, "Ev Idx": 88 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(771): getenv", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277043, "dur": 11, | |
| "args": { | |
| "Python parent id": 89, "Python id": 90, "Ev Idx": 89 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "_collections_abc.py(760): get", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277045, "dur": 8, | |
| "args": { | |
| "Python parent id": 90, "Python id": 91, "Ev Idx": 90 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(674): __getitem__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277046, "dur": 7, | |
| "args": { | |
| "Python parent id": 91, "Python id": 92, "Ev Idx": 91 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "os.py(754): encode", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277047, "dur": 4, | |
| "args": { | |
| "Python parent id": 92, "Python id": 93, "Ev Idx": 92 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277048, "dur": 0, | |
| "args": { | |
| "Python parent id": 93, "Python id": 94, "Ev Idx": 93 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method encode of str object at 0x7f9a1cbf2990>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277050, "dur": 1, | |
| "args": { | |
| "Python parent id": 93, "Python id": 95, "Ev Idx": 94 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDeviceCount>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277057, "dur": 1, | |
| "args": { | |
| "Python parent id": 86, "Python id": 96, "Ev Idx": 95 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method lower of str object at 0x7f9aa89848b0>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277060, "dur": 1, | |
| "args": { | |
| "Python parent id": 84, "Python id": 97, "Ev Idx": 96 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/_utils.py(735): <lambda>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277062, "dur": 18, | |
| "args": { | |
| "Python parent id": 84, "Python id": 98, "Ev Idx": 97 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(776): current_device", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277063, "dur": 17, | |
| "args": { | |
| "Python parent id": 98, "Python id": 99, "Ev Idx": 98 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277063, "dur": 1, | |
| "args": { | |
| "Python parent id": 99, "Python id": 100, "Ev Idx": 99 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277063, "dur": 1, | |
| "args": { | |
| "Python parent id": 100, "Python id": 101, "Ev Idx": 100 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277064, "dur": 0, | |
| "args": { | |
| "Python parent id": 101, "Python id": 102, "Ev Idx": 101 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDevice>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277066, "dur": 14, | |
| "args": { | |
| "Python parent id": 99, "Python id": 103, "Ev Idx": 102 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(364): __enter__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277084, "dur": 5, | |
| "args": { | |
| "Python parent id": 67, "Python id": 104, "Ev Idx": 103 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_exchangeDevice>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277086, "dur": 3, | |
| "args": { | |
| "Python parent id": 104, "Python id": 105, "Ev Idx": 104 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_synchronize>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277091, "dur": 22, | |
| "args": { | |
| "Python parent id": 67, "Python id": 106, "Ev Idx": 105 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(367): __exit__", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277113, "dur": 13, | |
| "args": { | |
| "Python parent id": 67, "Python id": 107, "Ev Idx": 106 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_maybeExchangeDevice>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277116, "dur": 1, | |
| "args": { | |
| "Python parent id": 107, "Python id": 108, "Ev Idx": 107 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "python_function", "name": "<built-in method _disable_profiler of PyCapsule object at 0x7f99f72d32d0>", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277120, "dur": 6, | |
| "args": { | |
| "Python parent id": 107, "Python id": 109, "Ev Idx": 108 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "cuda_runtime", "name": "cudaDeviceSynchronize", "pid": 3864658, "tid": 3864658, | |
| "ts": 1733485661277092, "dur": 20, | |
| "args": { | |
| "External id": 2490496, | |
| "cbid": 165, "correlation": 2490496 | |
| } | |
| }, | |
| { | |
| "ph": "s", "id": 2490496, "pid": 3864658, "tid": 3864658, "ts": 1733485661277092, | |
| "cat": "ac2g", "name": "ac2g" | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0, | |
| "args": { | |
| "labels": "CPU" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0, | |
| "args": { | |
| "sort_index": 3864658 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0, | |
| "args": { | |
| "labels": "GPU 0" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0, | |
| "args": { | |
| "sort_index": 16777216 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0, | |
| "args": { | |
| "labels": "GPU 1" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0, | |
| "args": { | |
| "sort_index": 16777217 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0, | |
| "args": { | |
| "labels": "GPU 2" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0, | |
| "args": { | |
| "sort_index": 16777218 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0, | |
| "args": { | |
| "labels": "GPU 3" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0, | |
| "args": { | |
| "sort_index": 16777219 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0, | |
| "args": { | |
| "labels": "GPU 4" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0, | |
| "args": { | |
| "sort_index": 16777220 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0, | |
| "args": { | |
| "labels": "GPU 5" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0, | |
| "args": { | |
| "sort_index": 16777221 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0, | |
| "args": { | |
| "labels": "GPU 6" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0, | |
| "args": { | |
| "sort_index": 16777222 | |
| } | |
| }, | |
| { | |
| "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0, | |
| "args": { | |
| "name": "python" | |
| } | |
| }, | |
| { | |
| "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0, | |
| "args": { | |
| "labels": "GPU 7" | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0, | |
| "args": { | |
| "sort_index": 16777223 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0, | |
| "args": { | |
| "name": "thread 0 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0, | |
| "args": { | |
| "sort_index": 0 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658, | |
| "args": { | |
| "name": "thread 3864658 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658, | |
| "args": { | |
| "sort_index": 3864658 | |
| } | |
| }, | |
| { | |
| "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658, | |
| "args": { | |
| "name": "thread 3864658 (pt_main_thread)" | |
| } | |
| }, | |
| { | |
| "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658, | |
| "args": { | |
| "sort_index": 3864658 | |
| } | |
| }, | |
| { | |
| "ph": "X", "cat": "Trace", "ts": 1733485661274744, "dur": 2382, | |
| "pid": "Spans", "tid": "PyTorch Profiler", | |
| "name": "PyTorch Profiler (0)", | |
| "args": { | |
| "Op count": 0 | |
| } | |
| }, | |
| { | |
| "name": "process_sort_index", "ph": "M", "ts": 1733485661274744, | |
| "pid": "Spans", "tid": 0, | |
| "args": { | |
| "sort_index": 536870912 | |
| } | |
| }, | |
| { | |
| "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g", | |
| "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1733485661274744 | |
| }, | |
| { | |
| "name": "Record Window End", "ph": "i", "s": "g", | |
| "pid": "", "tid": "", "ts": 1733485661278402 | |
| } | |
| ], | |
| "traceName": "/home/datta0/models/lora_final/profile_trace.json" | |
| } |