{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA A100-PCIE-40GB", "totalGlobalMem": 42411098112,
      "computeMajor": 8, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "regsPerMultiprocessor": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "sharedMemPerMultiprocessor": 167936,
      "numSms": 108, "sharedMemPerBlockOptin": 166912
    }
  ],
  "with_modules": 1,
  "record_shapes": 1,
  "with_stack": 1,
  "profile_memory": 1,
  "traceEvents": [
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276743, "dur": 383,
    "args": {
      "Python parent id": null, "Python id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276744, "dur": 382,
    "args": {
      "Python parent id": 1, "Python id": 2, "Ev Idx": 1
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733485661276746, "dur": 380,
    "args": {
      "Python parent id": 2, "Python id": 3, "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(323): check_internal_messages", "pid": 0, "tid": 0,
    "ts": 1733485661276746, "dur": 380,
    "args": {
      "Python parent id": 3, "Python id": 4, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733485661276748, "dur": 378,
    "args": {
      "Python parent id": 4, "Python id": 5, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276749, "dur": 377,
    "args": {
      "Python parent id": 5, "Python id": 6, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276751, "dur": 375,
    "args": {
      "Python parent id": 6, "Python id": 7, "Ev Idx": 6
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276753, "dur": 373,
    "args": {
      "Python parent id": 7, "Python id": 8, "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276753, "dur": 373,
    "args": {
      "Python parent id": 8, "Python id": 9, "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733485661276754, "dur": 372,
    "args": {
      "Python parent id": 9, "Python id": 10, "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(279): check_network_status", "pid": 0, "tid": 0,
    "ts": 1733485661276754, "dur": 372,
    "args": {
      "Python parent id": 10, "Python id": 11, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733485661276755, "dur": 371,
    "args": {
      "Python parent id": 11, "Python id": 12, "Ev Idx": 11
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276756, "dur": 370,
    "args": {
      "Python parent id": 12, "Python id": 13, "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276756, "dur": 370,
    "args": {
      "Python parent id": 13, "Python id": 14, "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276762, "dur": 364,
    "args": {
      "Python parent id": 14, "Python id": 15, "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276762, "dur": 364,
    "args": {
      "Python parent id": 15, "Python id": 16, "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733485661276763, "dur": 363,
    "args": {
      "Python parent id": 16, "Python id": 17, "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(303): check_stop_status", "pid": 0, "tid": 0,
    "ts": 1733485661276764, "dur": 362,
    "args": {
      "Python parent id": 17, "Python id": 18, "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/wandb_run.py(261): _loop_check_status", "pid": 0, "tid": 0,
    "ts": 1733485661276764, "dur": 362,
    "args": {
      "Python parent id": 18, "Python id": 19, "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276765, "dur": 361,
    "args": {
      "Python parent id": 19, "Python id": 20, "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276765, "dur": 361,
    "args": {
      "Python parent id": 20, "Python id": 21, "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276770, "dur": 356,
    "args": {
      "Python parent id": 21, "Python id": 22, "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276770, "dur": 356,
    "args": {
      "Python parent id": 22, "Python id": 23, "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(917): run", "pid": 0, "tid": 0,
    "ts": 1733485661276771, "dur": 355,
    "args": {
      "Python parent id": 23, "Python id": 24, "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router.py(70): message_loop", "pid": 0, "tid": 0,
    "ts": 1733485661276771, "dur": 355,
    "args": {
      "Python parent id": 24, "Python id": 25, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/interface/router_sock.py(27): _read_message", "pid": 0, "tid": 0,
    "ts": 1733485661276772, "dur": 354,
    "args": {
      "Python parent id": 25, "Python id": 26, "Ev Idx": 25
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(285): read_server_response", "pid": 0, "tid": 0,
    "ts": 1733485661276773, "dur": 353,
    "args": {
      "Python parent id": 26, "Python id": 27, "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "wandb/sdk/lib/sock_client.py(255): _read_packet_bytes", "pid": 0, "tid": 0,
    "ts": 1733485661276774, "dur": 352,
    "args": {
      "Python parent id": 27, "Python id": 28, "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276779, "dur": 347,
    "args": {
      "Python parent id": 28, "Python id": 29, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276779, "dur": 347,
    "args": {
      "Python parent id": 29, "Python id": 30, "Ev Idx": 29
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0,
    "ts": 1733485661276780, "dur": 346,
    "args": {
      "Python parent id": 30, "Python id": 31, "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276780, "dur": 346,
    "args": {
      "Python parent id": 31, "Python id": 32, "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276781, "dur": 345,
    "args": {
      "Python parent id": 32, "Python id": 33, "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(937): _bootstrap", "pid": 0, "tid": 0,
    "ts": 1733485661276784, "dur": 342,
    "args": {
      "Python parent id": 33, "Python id": 34, "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(980): _bootstrap_inner", "pid": 0, "tid": 0,
    "ts": 1733485661276784, "dur": 342,
    "args": {
      "Python parent id": 34, "Python id": 35, "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "tqdm/_monitor.py(60): run", "pid": 0, "tid": 0,
    "ts": 1733485661276784, "dur": 342,
    "args": {
      "Python parent id": 35, "Python id": 36, "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(581): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276785, "dur": 341,
    "args": {
      "Python parent id": 36, "Python id": 37, "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "threading.py(316): wait", "pid": 0, "tid": 0,
    "ts": 1733485661276785, "dur": 341,
    "args": {
      "Python parent id": 37, "Python id": 38, "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "ft_llm.py(298): <module>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276789, "dur": 337,
    "args": {
      "Python parent id": null, "Python id": 39, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "ft_llm.py(260): train", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276790, "dur": 336,
    "args": {
      "Python parent id": 39, "Python id": 40, "Ev Idx": 39
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(699): __exit__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276792, "dur": 334,
    "args": {
      "Python parent id": 40, "Python id": 41, "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(715): stop", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276793, "dur": 333,
    "args": {
      "Python parent id": 41, "Python id": 42, "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(744): _transit_action", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276793, "dur": 333,
    "args": {
      "Python parent id": 42, "Python id": 43, "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(161): start_trace", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276794, "dur": 197,
    "args": {
      "Python parent id": 43, "Python id": 44, "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(289): _start_trace", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276795, "dur": 74,
    "args": {
      "Python parent id": 44, "Python id": 45, "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276875, "dur": 27,
    "args": {
      "Python parent id": 44, "Python id": 46, "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276896, "dur": 5,
    "args": {
      "Python parent id": 46, "Python id": 47, "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276904, "dur": 2,
    "args": {
      "Python parent id": 44, "Python id": 48, "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276904, "dur": 2,
    "args": {
      "Python parent id": 48, "Python id": 49, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276907, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 50, "Ev Idx": 49
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276907, "dur": 0,
    "args": {
      "Python parent id": 50, "Python id": 51, "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(264): add_metadata_json", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276908, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 52, "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _add_metadata_json of PyCapsule object at 0x7f99f72d3330>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276908, "dur": 0,
    "args": {
      "Python parent id": 52, "Python id": 53, "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method kineto_available of PyCapsule object at 0x7f99f72d3390>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276911, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 54, "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(279): _get_distributed_info", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276913, "dur": 41,
    "args": {
      "Python parent id": 44, "Python id": 55, "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/__init__.py(9): is_available", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276921, "dur": 23,
    "args": {
      "Python parent id": 55, "Python id": 56, "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276943, "dur": 1,
    "args": {
      "Python parent id": 56, "Python id": 57, "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(973): is_initialized", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276947, "dur": 7,
    "args": {
      "Python parent id": 55, "Python id": 58, "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(588): WORLD", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276949, "dur": 4,
    "args": {
      "Python parent id": 58, "Python id": 59, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/distributed/distributed_c10d.py(460): default_pg", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276953, "dur": 0,
    "args": {
      "Python parent id": 59, "Python id": 60, "Ev Idx": 59
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276955, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 61, "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(356): __getattr__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276978, "dur": 8,
    "args": {
      "Python parent id": 44, "Python id": 62, "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/utils/_config_module.py(140): __getattr__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276984, "dur": 2,
    "args": {
      "Python parent id": 62, "Python id": 63, "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method items of dict object at 0x7f9995dba9c0>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276990, "dur": 0,
    "args": {
      "Python parent id": 44, "Python id": 64, "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/profiler/profiler.py(195): stop_trace", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276993, "dur": 133,
    "args": {
      "Python parent id": 43, "Python id": 65, "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/autograd/profiler.py(291): __exit__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276995, "dur": 131,
    "args": {
      "Python parent id": 65, "Python id": 66, "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(782): synchronize", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661276999, "dur": 127,
    "args": {
      "Python parent id": 66, "Python id": 67, "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277001, "dur": 6,
    "args": {
      "Python parent id": 67, "Python id": 68, "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277003, "dur": 4,
    "args": {
      "Python parent id": 68, "Python id": 69, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277005, "dur": 2,
    "args": {
      "Python parent id": 69, "Python id": 70, "Ev Idx": 69
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(360): __init__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277011, "dur": 72,
    "args": {
      "Python parent id": 67, "Python id": 71, "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/_utils.py(9): _get_device_index", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277013, "dur": 68,
    "args": {
      "Python parent id": 71, "Python id": 72, "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277014, "dur": 1,
    "args": {
      "Python parent id": 72, "Python id": 73, "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277015, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 74, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277016, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 75, "Ev Idx": 74
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277019, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 76, "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277020, "dur": 0,
    "args": {
      "Python parent id": 72, "Python id": 77, "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(759): _get_device_index", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277026, "dur": 55,
    "args": {
      "Python parent id": 72, "Python id": 78, "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277027, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 79, "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277028, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 80, "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277028, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 81, "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_jit_internal.py(1120): is_scripting", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277029, "dur": 0,
    "args": {
      "Python parent id": 78, "Python id": 82, "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(733): _get_current_device_index", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277030, "dur": 51,
    "args": {
      "Python parent id": 78, "Python id": 83, "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(721): _get_device_attr", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277032, "dur": 48,
    "args": {
      "Python parent id": 83, "Python id": 84, "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(708): _get_available_device_type", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277033, "dur": 25,
    "args": {
      "Python parent id": 84, "Python id": 85, "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(105): is_available", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277035, "dur": 23,
    "args": {
      "Python parent id": 85, "Python id": 86, "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(96): _is_compiled", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277038, "dur": 2,
    "args": {
      "Python parent id": 86, "Python id": 87, "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function hasattr>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277040, "dur": 0,
    "args": {
      "Python parent id": 87, "Python id": 88, "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(101): _nvml_based_avail", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277041, "dur": 13,
    "args": {
      "Python parent id": 86, "Python id": 89, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(771): getenv", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277043, "dur": 11,
    "args": {
      "Python parent id": 89, "Python id": 90, "Ev Idx": 89
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "_collections_abc.py(760): get", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277045, "dur": 8,
    "args": {
      "Python parent id": 90, "Python id": 91, "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(674): __getitem__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277046, "dur": 7,
    "args": {
      "Python parent id": 91, "Python id": 92, "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "os.py(754): encode", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277047, "dur": 4,
    "args": {
      "Python parent id": 92, "Python id": 93, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function isinstance>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277048, "dur": 0,
    "args": {
      "Python parent id": 93, "Python id": 94, "Ev Idx": 93
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method encode of str object at 0x7f9a1cbf2990>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277050, "dur": 1,
    "args": {
      "Python parent id": 93, "Python id": 95, "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDeviceCount>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277057, "dur": 1,
    "args": {
      "Python parent id": 86, "Python id": 96, "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method lower of str object at 0x7f9aa89848b0>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277060, "dur": 1,
    "args": {
      "Python parent id": 84, "Python id": 97, "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/_utils.py(735): <lambda>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277062, "dur": 18,
    "args": {
      "Python parent id": 84, "Python id": 98, "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(776): current_device", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277063, "dur": 17,
    "args": {
      "Python parent id": 98, "Python id": 99, "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(263): _lazy_init", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277063, "dur": 1,
    "args": {
      "Python parent id": 99, "Python id": 100, "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(216): is_initialized", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277063, "dur": 1,
    "args": {
      "Python parent id": 100, "Python id": 101, "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_isInBadFork>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277064, "dur": 0,
    "args": {
      "Python parent id": 101, "Python id": 102, "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_getDevice>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277066, "dur": 14,
    "args": {
      "Python parent id": 99, "Python id": 103, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(364): __enter__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277084, "dur": 5,
    "args": {
      "Python parent id": 67, "Python id": 104, "Ev Idx": 103
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_exchangeDevice>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277086, "dur": 3,
    "args": {
      "Python parent id": 104, "Python id": 105, "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_synchronize>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277091, "dur": 22,
    "args": {
      "Python parent id": 67, "Python id": 106, "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "torch/cuda/__init__.py(367): __exit__", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277113, "dur": 13,
    "args": {
      "Python parent id": 67, "Python id": 107, "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in function _cuda_maybeExchangeDevice>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277116, "dur": 1,
    "args": {
      "Python parent id": 107, "Python id": 108, "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "python_function", "name": "<built-in method _disable_profiler of PyCapsule object at 0x7f99f72d32d0>", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277120, "dur": 6,
    "args": {
      "Python parent id": 107, "Python id": 109, "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cuda_runtime", "name": "cudaDeviceSynchronize", "pid": 3864658, "tid": 3864658,
    "ts": 1733485661277092, "dur": 20,
    "args": {
      "External id": 2490496,
      "cbid": 165, "correlation": 2490496
    }
  },
  {
    "ph": "s", "id": 2490496, "pid": 3864658, "tid": 3864658, "ts": 1733485661277092,
    "cat": "ac2g", "name": "ac2g"
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 0,
    "args": {
      "sort_index": 3864658
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0,
    "args": {
      "labels": "GPU 0"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0,
    "args": {
      "sort_index": 16777216
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0,
    "args": {
      "labels": "GPU 1"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 1, "tid": 0,
    "args": {
      "sort_index": 16777217
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0,
    "args": {
      "labels": "GPU 2"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 2, "tid": 0,
    "args": {
      "sort_index": 16777218
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0,
    "args": {
      "labels": "GPU 3"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3, "tid": 0,
    "args": {
      "sort_index": 16777219
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0,
    "args": {
      "labels": "GPU 4"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 4, "tid": 0,
    "args": {
      "sort_index": 16777220
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0,
    "args": {
      "labels": "GPU 5"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 5, "tid": 0,
    "args": {
      "sort_index": 16777221
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0,
    "args": {
      "labels": "GPU 6"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 6, "tid": 0,
    "args": {
      "sort_index": 16777222
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0,
    "args": {
      "name": "python"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0,
    "args": {
      "labels": "GPU 7"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 7, "tid": 0,
    "args": {
      "sort_index": 16777223
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0,
    "args": {
      "name": "thread 0 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 0, "tid": 0,
    "args": {
      "sort_index": 0
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658,
    "args": {
      "name": "thread 3864658 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658,
    "args": {
      "sort_index": 3864658
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658,
    "args": {
      "name": "thread 3864658 (pt_main_thread)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1733485661276863, "pid": 3864658, "tid": 3864658,
    "args": {
      "sort_index": 3864658
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1733485661274744, "dur": 2382,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1733485661274744,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1733485661274744
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1733485661278402
  }
  ],
  "traceName": "/home/datta0/models/lora_final/profile_trace.json"
}