pushing model
Browse files- .gitattributes +1 -0
- Breakout-base.cleanrl_model +3 -0
- README.md +85 -0
- dqn_atari.py +357 -0
- events.out.tfevents.1685407342.ccaa1a6bd80b.5479.0 +3 -0
- poetry.lock +0 -0
- pyproject.toml +11 -0
- replay.mp4 +0 -0
- videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-0.mp4 +0 -0
- videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-1.mp4 +0 -0
- videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-8.mp4 +0 -0
.gitattributes
CHANGED
@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
32 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
32 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
33 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
34 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
35 |
+
Breakout-base.cleanrl_model filter=lfs diff=lfs merge=lfs -text
|
Breakout-base.cleanrl_model
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:62e622e13ca7282ae18a0dfd9274f3393e0edd2d873dae853b21d9488266cc1c
|
3 |
+
size 6747719
|
README.md
ADDED
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
---
|
2 |
+
tags:
|
3 |
+
- BreakoutNoFrameskip-v4
|
4 |
+
- deep-reinforcement-learning
|
5 |
+
- reinforcement-learning
|
6 |
+
- custom-implementation
|
7 |
+
library_name: cleanrl
|
8 |
+
model-index:
|
9 |
+
- name: DQN
|
10 |
+
results:
|
11 |
+
- task:
|
12 |
+
type: reinforcement-learning
|
13 |
+
name: reinforcement-learning
|
14 |
+
dataset:
|
15 |
+
name: BreakoutNoFrameskip-v4
|
16 |
+
type: BreakoutNoFrameskip-v4
|
17 |
+
metrics:
|
18 |
+
- type: mean_reward
|
19 |
+
value: 221.40 +/- 32.78
|
20 |
+
name: mean_reward
|
21 |
+
verified: false
|
22 |
+
---
|
23 |
+
|
24 |
+
# (CleanRL) **DQN** Agent Playing **BreakoutNoFrameskip-v4**
|
25 |
+
|
26 |
+
This is a trained model of a DQN agent playing BreakoutNoFrameskip-v4.
|
27 |
+
The model was trained by using [CleanRL](https://github.com/vwxyzjn/cleanrl) and the most up-to-date training code can be
|
28 |
+
found [here](https://github.com/vwxyzjn/cleanrl/blob/master/cleanrl/dqn_atari.py).
|
29 |
+
|
30 |
+
## Get Started
|
31 |
+
|
32 |
+
To use this model, please install the `cleanrl` package with the following command:
|
33 |
+
|
34 |
+
```
|
35 |
+
pip install "cleanrl[dqn_atari]"
|
36 |
+
python -m cleanrl_utils.enjoy --exp-name dqn_atari --env-id BreakoutNoFrameskip-v4
|
37 |
+
```
|
38 |
+
|
39 |
+
Please refer to the [documentation](https://docs.cleanrl.dev/get-started/zoo/) for more detail.
|
40 |
+
|
41 |
+
|
42 |
+
## Command to reproduce the training
|
43 |
+
|
44 |
+
```bash
|
45 |
+
curl -OL https://huggingface.co/odiaz1066/BreakoutNoFrameskip-v4-dqn_atari-seed1/raw/main/dqn_atari.py
|
46 |
+
curl -OL https://huggingface.co/odiaz1066/BreakoutNoFrameskip-v4-dqn_atari-seed1/raw/main/pyproject.toml
|
47 |
+
curl -OL https://huggingface.co/odiaz1066/BreakoutNoFrameskip-v4-dqn_atari-seed1/raw/main/poetry.lock
|
48 |
+
poetry install --all-extras
|
49 |
+
python dqn_atari.py --track --save-model --capture-video --resume --total-timesteps 0 --upload-model --seed 1 --hf-entity odiaz1066
|
50 |
+
```
|
51 |
+
|
52 |
+
# Hyperparameters
|
53 |
+
```python
|
54 |
+
{'batch_size': 32,
|
55 |
+
'buffer_size': 1000000,
|
56 |
+
'capture_video': True,
|
57 |
+
'checkpoint': False,
|
58 |
+
'checkpoint_frequency': 100000,
|
59 |
+
'cuda': True,
|
60 |
+
'end_e': 0.01,
|
61 |
+
'env_id': 'BreakoutNoFrameskip-v4',
|
62 |
+
'exp_name': 'dqn_atari',
|
63 |
+
'exploration_fraction': 0.1,
|
64 |
+
'gamma': 0.99,
|
65 |
+
'hf_entity': 'odiaz1066',
|
66 |
+
'initial_steps': 0,
|
67 |
+
'learning_rate': 0.0001,
|
68 |
+
'learning_starts': 80000,
|
69 |
+
'num_envs': 1,
|
70 |
+
'resume': True,
|
71 |
+
'rotate': False,
|
72 |
+
'save_model': True,
|
73 |
+
'seed': 1,
|
74 |
+
'start_e': 1,
|
75 |
+
'target_network_frequency': 1000,
|
76 |
+
'tau': 1.0,
|
77 |
+
'torch_deterministic': True,
|
78 |
+
'total_timesteps': 0,
|
79 |
+
'track': True,
|
80 |
+
'train_frequency': 4,
|
81 |
+
'upload_model': True,
|
82 |
+
'wandb_entity': None,
|
83 |
+
'wandb_project_name': 'lagomorph'}
|
84 |
+
```
|
85 |
+
|
dqn_atari.py
ADDED
@@ -0,0 +1,357 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# docs and experiment results can be found at https://docs.cleanrl.dev/rl-algorithms/dqn/#dqn_ataripy
|
2 |
+
import argparse
|
3 |
+
import os
|
4 |
+
import random
|
5 |
+
import time
|
6 |
+
from distutils.util import strtobool
|
7 |
+
|
8 |
+
import gymnasium as gym
|
9 |
+
import numpy as np
|
10 |
+
import torch
|
11 |
+
import torch.nn as nn
|
12 |
+
import torch.nn.functional as F
|
13 |
+
import torch.optim as optim
|
14 |
+
from stable_baselines3.common.atari_wrappers import (
|
15 |
+
ClipRewardEnv,
|
16 |
+
EpisodicLifeEnv,
|
17 |
+
FireResetEnv,
|
18 |
+
MaxAndSkipEnv,
|
19 |
+
NoopResetEnv,
|
20 |
+
)
|
21 |
+
from stable_baselines3.common.buffers import ReplayBuffer
|
22 |
+
from torch.utils.tensorboard import SummaryWriter
|
23 |
+
from typing import Callable
|
24 |
+
from dotenv import load_dotenv
|
25 |
+
import sys
|
26 |
+
sys.path.append('.')
|
27 |
+
from wrappers.rotation import RotateObservation
|
28 |
+
|
29 |
+
def parse_args():
|
30 |
+
# fmt: off
|
31 |
+
parser = argparse.ArgumentParser()
|
32 |
+
parser.add_argument("--exp-name", type=str, default=os.path.basename(__file__).rstrip(".py"),
|
33 |
+
help="the name of this experiment")
|
34 |
+
parser.add_argument("--seed", type=int, default=1,
|
35 |
+
help="seed of the experiment")
|
36 |
+
parser.add_argument("--torch-deterministic", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
|
37 |
+
help="if toggled, `torch.backends.cudnn.deterministic=False`")
|
38 |
+
parser.add_argument("--cuda", type=lambda x: bool(strtobool(x)), default=True, nargs="?", const=True,
|
39 |
+
help="if toggled, cuda will be enabled by default")
|
40 |
+
parser.add_argument("--track", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
41 |
+
help="if toggled, this experiment will be tracked with Weights and Biases")
|
42 |
+
parser.add_argument("--wandb-project-name", type=str, default="lagomorph",
|
43 |
+
help="the wandb's project name")
|
44 |
+
parser.add_argument("--wandb-entity", type=str, default=None,
|
45 |
+
help="the entity (team) of wandb's project")
|
46 |
+
parser.add_argument("--capture-video", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
47 |
+
help="whether to capture videos of the agent performances (check out `videos` folder)")
|
48 |
+
parser.add_argument("--save-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
49 |
+
help="whether to save model into the `runs/{run_name}` folder")
|
50 |
+
parser.add_argument("--upload-model", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
51 |
+
help="whether to upload the saved model to huggingface")
|
52 |
+
parser.add_argument("--hf-entity", type=str, default="",
|
53 |
+
help="the user or org name of the model repository from the Hugging Face Hub")
|
54 |
+
|
55 |
+
# Algorithm specific arguments
|
56 |
+
parser.add_argument("--env-id", type=str, default="BreakoutNoFrameskip-v4",
|
57 |
+
help="the id of the environment")
|
58 |
+
parser.add_argument("--total-timesteps", type=int, default=10000000,
|
59 |
+
help="total timesteps of the experiments")
|
60 |
+
parser.add_argument("--learning-rate", type=float, default=1e-4,
|
61 |
+
help="the learning rate of the optimizer")
|
62 |
+
parser.add_argument("--num-envs", type=int, default=1,
|
63 |
+
help="the number of parallel game environments")
|
64 |
+
parser.add_argument("--buffer-size", type=int, default=1000000,
|
65 |
+
help="the replay memory buffer size")
|
66 |
+
parser.add_argument("--gamma", type=float, default=0.99,
|
67 |
+
help="the discount factor gamma")
|
68 |
+
parser.add_argument("--tau", type=float, default=1.,
|
69 |
+
help="the target network update rate")
|
70 |
+
parser.add_argument("--target-network-frequency", type=int, default=1000,
|
71 |
+
help="the timesteps it takes to update the target network")
|
72 |
+
parser.add_argument("--batch-size", type=int, default=32,
|
73 |
+
help="the batch size of sample from the reply memory")
|
74 |
+
parser.add_argument("--start-e", type=float, default=1,
|
75 |
+
help="the starting epsilon for exploration")
|
76 |
+
parser.add_argument("--end-e", type=float, default=0.01,
|
77 |
+
help="the ending epsilon for exploration")
|
78 |
+
parser.add_argument("--exploration-fraction", type=float, default=0.10,
|
79 |
+
help="the fraction of `total-timesteps` it takes from start-e to go end-e")
|
80 |
+
parser.add_argument("--learning-starts", type=int, default=80000,
|
81 |
+
help="timestep to start learning")
|
82 |
+
parser.add_argument("--train-frequency", type=int, default=4,
|
83 |
+
help="the frequency of training")
|
84 |
+
parser.add_argument("--checkpoint", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
85 |
+
help="whether to checkpoint the model")
|
86 |
+
parser.add_argument("--checkpoint-frequency", type=int, default=100000,
|
87 |
+
help="the frequency of saving checkpoint")
|
88 |
+
parser.add_argument("--resume", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
89 |
+
help="resume training from the last checkpoint")
|
90 |
+
parser.add_argument("--initial-steps", type=int, default=0,
|
91 |
+
help="the number of steps that were done in previous training runs")
|
92 |
+
parser.add_argument("--rotate", type=lambda x: bool(strtobool(x)), default=False, nargs="?", const=True,
|
93 |
+
help="whether to rotate the observation sometimes")
|
94 |
+
args = parser.parse_args()
|
95 |
+
# fmt: on
|
96 |
+
assert args.num_envs == 1, "vectorized envs are not supported at the moment"
|
97 |
+
|
98 |
+
return args
|
99 |
+
|
100 |
+
|
101 |
+
def make_env(env_id, seed, idx, capture_video, run_name):
|
102 |
+
def thunk():
|
103 |
+
if capture_video and idx == 0:
|
104 |
+
env = gym.make(env_id, render_mode="rgb_array")
|
105 |
+
env = gym.wrappers.RecordVideo(env, f"videos/{run_name}")
|
106 |
+
else:
|
107 |
+
env = gym.make(env_id)
|
108 |
+
env = gym.wrappers.RecordEpisodeStatistics(env)
|
109 |
+
env = NoopResetEnv(env, noop_max=30)
|
110 |
+
env = MaxAndSkipEnv(env, skip=4)
|
111 |
+
env = EpisodicLifeEnv(env)
|
112 |
+
if "FIRE" in env.unwrapped.get_action_meanings():
|
113 |
+
env = FireResetEnv(env)
|
114 |
+
env = ClipRewardEnv(env)
|
115 |
+
env = gym.wrappers.ResizeObservation(env, (84, 84))
|
116 |
+
env = gym.wrappers.GrayScaleObservation(env)
|
117 |
+
if args.rotate:
|
118 |
+
env = RotateObservation(env)
|
119 |
+
env = gym.wrappers.FrameStack(env, 4)
|
120 |
+
env.action_space.seed(seed)
|
121 |
+
|
122 |
+
return env
|
123 |
+
|
124 |
+
return thunk
|
125 |
+
|
126 |
+
|
127 |
+
# ALGO LOGIC: initialize agent here:
|
128 |
+
class QNetwork(nn.Module):
|
129 |
+
def __init__(self, env):
|
130 |
+
super().__init__()
|
131 |
+
self.network = nn.Sequential(
|
132 |
+
nn.Conv2d(4, 32, 8, stride=4),
|
133 |
+
nn.ReLU(),
|
134 |
+
nn.Conv2d(32, 64, 4, stride=2),
|
135 |
+
nn.ReLU(),
|
136 |
+
nn.Conv2d(64, 64, 3, stride=1),
|
137 |
+
nn.ReLU(),
|
138 |
+
nn.Flatten(),
|
139 |
+
nn.Linear(3136, 512),
|
140 |
+
nn.ReLU(),
|
141 |
+
nn.Linear(512, env.single_action_space.n),
|
142 |
+
)
|
143 |
+
|
144 |
+
def forward(self, x):
|
145 |
+
return self.network(x / 255.0)
|
146 |
+
|
147 |
+
|
148 |
+
def linear_schedule(start_e: float, end_e: float, duration: int, t: int):
|
149 |
+
slope = (end_e - start_e) / duration
|
150 |
+
return max(slope * t + start_e, end_e)
|
151 |
+
|
152 |
+
def evaluate(
|
153 |
+
model_path: str,
|
154 |
+
make_env: Callable,
|
155 |
+
env_id: str,
|
156 |
+
eval_episodes: int,
|
157 |
+
run_name: str,
|
158 |
+
Model: torch.nn.Module,
|
159 |
+
device: torch.device = torch.device("cpu"),
|
160 |
+
epsilon: float = 0.05,
|
161 |
+
capture_video: bool = True,
|
162 |
+
):
|
163 |
+
envs = gym.vector.SyncVectorEnv([make_env(env_id, 0, 0, capture_video, run_name)])
|
164 |
+
model = Model(envs).to(device)
|
165 |
+
model.load_state_dict(torch.load(model_path, map_location=device))
|
166 |
+
model.eval()
|
167 |
+
|
168 |
+
obs, _ = envs.reset()
|
169 |
+
episodic_returns = []
|
170 |
+
while len(episodic_returns) < eval_episodes:
|
171 |
+
if random.random() < epsilon:
|
172 |
+
actions = np.array([envs.single_action_space.sample() for _ in range(envs.num_envs)])
|
173 |
+
else:
|
174 |
+
q_values = model(torch.Tensor(obs).to(device))
|
175 |
+
actions = torch.argmax(q_values, dim=1).cpu().numpy()
|
176 |
+
next_obs, _, _, _, infos = envs.step(actions)
|
177 |
+
if "final_info" in infos:
|
178 |
+
for info in infos["final_info"]:
|
179 |
+
if "episode" in info:
|
180 |
+
print(f"eval_episode={len(episodic_returns)}, episodic_return={info['episode']['r']}")
|
181 |
+
episodic_returns += [info["episode"]["r"]]
|
182 |
+
obs = next_obs
|
183 |
+
|
184 |
+
return episodic_returns
|
185 |
+
|
186 |
+
if __name__ == "__main__":
|
187 |
+
import stable_baselines3 as sb3
|
188 |
+
|
189 |
+
if sb3.__version__ < "2.0":
|
190 |
+
raise ValueError(
|
191 |
+
"""Ongoing migration: run the following command to install the new dependencies:
|
192 |
+
|
193 |
+
poetry run pip install "stable_baselines3==2.0.0a1" "gymnasium[atari,accept-rom-license]==0.28.1" "ale-py==0.8.1"
|
194 |
+
"""
|
195 |
+
)
|
196 |
+
args = parse_args()
|
197 |
+
load_dotenv()
|
198 |
+
run_name = f"{args.env_id}__{args.exp_name}__{args.seed}__{int(time.time())}"
|
199 |
+
if args.track:
|
200 |
+
import wandb
|
201 |
+
|
202 |
+
wandb.init(
|
203 |
+
project=args.wandb_project_name,
|
204 |
+
entity=args.wandb_entity,
|
205 |
+
sync_tensorboard=True,
|
206 |
+
config=vars(args),
|
207 |
+
name=run_name,
|
208 |
+
monitor_gym=True,
|
209 |
+
save_code=True,
|
210 |
+
)
|
211 |
+
writer = SummaryWriter(f"runs/{run_name}")
|
212 |
+
writer.add_text(
|
213 |
+
"hyperparameters",
|
214 |
+
"|param|value|\n|-|-|\n%s" % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
|
215 |
+
)
|
216 |
+
|
217 |
+
# TRY NOT TO MODIFY: seeding
|
218 |
+
random.seed(args.seed)
|
219 |
+
np.random.seed(args.seed)
|
220 |
+
torch.manual_seed(args.seed)
|
221 |
+
torch.backends.cudnn.deterministic = args.torch_deterministic
|
222 |
+
|
223 |
+
device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
|
224 |
+
|
225 |
+
# env setup
|
226 |
+
envs = gym.vector.SyncVectorEnv(
|
227 |
+
[make_env(args.env_id, args.seed + i, i, args.capture_video, run_name) for i in range(args.num_envs)]
|
228 |
+
)
|
229 |
+
assert isinstance(envs.single_action_space, gym.spaces.Discrete), "only discrete action space is supported"
|
230 |
+
|
231 |
+
q_network = QNetwork(envs).to(device)
|
232 |
+
optimizer = optim.Adam(q_network.parameters(), lr=args.learning_rate)
|
233 |
+
target_network = QNetwork(envs).to(device)
|
234 |
+
target_network.load_state_dict(q_network.state_dict())
|
235 |
+
|
236 |
+
rb = ReplayBuffer(
|
237 |
+
args.buffer_size,
|
238 |
+
envs.single_observation_space,
|
239 |
+
envs.single_action_space,
|
240 |
+
device,
|
241 |
+
optimize_memory_usage=True,
|
242 |
+
handle_timeout_termination=False,
|
243 |
+
)
|
244 |
+
start_time = time.time()
|
245 |
+
|
246 |
+
if args.resume:
|
247 |
+
checkpoint_path = f"models/Breakout.checkpoint.pth"
|
248 |
+
q_network.load_state_dict(torch.load(checkpoint_path))
|
249 |
+
print(f"checkpoint loaded from {checkpoint_path}")
|
250 |
+
|
251 |
+
# TRY NOT TO MODIFY: start the game
|
252 |
+
obs, _ = envs.reset(seed=args.seed)
|
253 |
+
for global_step in range(args.initial_steps, args.total_timesteps):
|
254 |
+
# ALGO LOGIC: put action logic here
|
255 |
+
epsilon = linear_schedule(args.start_e, args.end_e, args.exploration_fraction * args.total_timesteps, global_step)
|
256 |
+
if random.random() < epsilon:
|
257 |
+
actions = np.array([envs.single_action_space.sample() for _ in range(envs.num_envs)])
|
258 |
+
else:
|
259 |
+
q_values = q_network(torch.Tensor(obs).to(device))
|
260 |
+
actions = torch.argmax(q_values, dim=1).cpu().numpy()
|
261 |
+
|
262 |
+
# TRY NOT TO MODIFY: execute the game and log data.
|
263 |
+
next_obs, rewards, terminated, truncated, infos = envs.step(actions)
|
264 |
+
|
265 |
+
# TRY NOT TO MODIFY: record rewards for plotting purposes
|
266 |
+
if "final_info" in infos:
|
267 |
+
for info in infos["final_info"]:
|
268 |
+
# Skip the envs that are not done
|
269 |
+
if "episode" not in info:
|
270 |
+
continue
|
271 |
+
print(f"global_step={global_step}, episodic_return={info['episode']['r']}")
|
272 |
+
writer.add_scalar("charts/episodic_return", info["episode"]["r"], global_step)
|
273 |
+
writer.add_scalar("charts/episodic_length", info["episode"]["l"], global_step)
|
274 |
+
writer.add_scalar("charts/epsilon", epsilon, global_step)
|
275 |
+
|
276 |
+
# TRY NOT TO MODIFY: save data to reply buffer; handle `final_observation`
|
277 |
+
real_next_obs = next_obs.copy()
|
278 |
+
for idx, d in enumerate(truncated):
|
279 |
+
if d:
|
280 |
+
real_next_obs[idx] = infos["final_observation"][idx]
|
281 |
+
rb.add(obs, real_next_obs, actions, rewards, terminated, infos)
|
282 |
+
|
283 |
+
# TRY NOT TO MODIFY: CRUCIAL step easy to overlook
|
284 |
+
obs = next_obs
|
285 |
+
|
286 |
+
# ALGO LOGIC: training.
|
287 |
+
if global_step > args.learning_starts:
|
288 |
+
if global_step % args.train_frequency == 0:
|
289 |
+
data = rb.sample(args.batch_size)
|
290 |
+
with torch.no_grad():
|
291 |
+
target_max, _ = target_network(data.next_observations).max(dim=1)
|
292 |
+
td_target = data.rewards.flatten() + args.gamma * target_max * (1 - data.dones.flatten())
|
293 |
+
old_val = q_network(data.observations).gather(1, data.actions).squeeze()
|
294 |
+
loss = F.mse_loss(td_target, old_val)
|
295 |
+
|
296 |
+
if global_step % 100 == 0:
|
297 |
+
writer.add_scalar("losses/td_loss", loss, global_step)
|
298 |
+
writer.add_scalar("losses/q_values", old_val.mean().item(), global_step)
|
299 |
+
print("SPS:", int(global_step / (time.time() - start_time)))
|
300 |
+
writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
|
301 |
+
|
302 |
+
# optimize the model
|
303 |
+
optimizer.zero_grad()
|
304 |
+
loss.backward()
|
305 |
+
optimizer.step()
|
306 |
+
|
307 |
+
# update target network
|
308 |
+
if global_step % args.target_network_frequency == 0:
|
309 |
+
for target_network_param, q_network_param in zip(target_network.parameters(), q_network.parameters()):
|
310 |
+
target_network_param.data.copy_(
|
311 |
+
args.tau * q_network_param.data + (1.0 - args.tau) * target_network_param.data
|
312 |
+
)
|
313 |
+
|
314 |
+
# save checkpoint
|
315 |
+
if (args.checkpoint):
|
316 |
+
if global_step % args.checkpoint_frequency == 0:
|
317 |
+
checkpoint_path = f"runs/{run_name}/{args.exp_name}.checkpoint.pth"
|
318 |
+
torch.save(q_network.state_dict(), checkpoint_path)
|
319 |
+
print(f"checkpoint saved to {checkpoint_path}")
|
320 |
+
if args.track:
|
321 |
+
artifact = wandb.Artifact(f"{args.exp_name}_checkpoint", "checkpoint")
|
322 |
+
artifact.add_file(checkpoint_path)
|
323 |
+
wandb.log_artifact(artifact)
|
324 |
+
|
325 |
+
|
326 |
+
if args.save_model:
|
327 |
+
model_path = f"runs/{run_name}/Breakout-base.cleanrl_model"
|
328 |
+
torch.save(q_network.state_dict(), model_path)
|
329 |
+
print(f"model saved to {model_path}")
|
330 |
+
if args.track:
|
331 |
+
artifact = wandb.Artifact(f"Breakout-base_model", "model")
|
332 |
+
artifact.add_file(model_path)
|
333 |
+
wandb.log_artifact(artifact)
|
334 |
+
|
335 |
+
episodic_returns = evaluate(
|
336 |
+
model_path,
|
337 |
+
make_env,
|
338 |
+
args.env_id,
|
339 |
+
eval_episodes=10,
|
340 |
+
run_name=f"{run_name}-eval",
|
341 |
+
Model=QNetwork,
|
342 |
+
device=device,
|
343 |
+
epsilon=0.05,
|
344 |
+
)
|
345 |
+
for idx, episodic_return in enumerate(episodic_returns):
|
346 |
+
writer.add_scalar("eval/episodic_return", episodic_return, idx)
|
347 |
+
|
348 |
+
if args.upload_model:
|
349 |
+
#TODO: Currently gives an error. *Maybe* I'll get around to fixing it once everything else works correctly.
|
350 |
+
from utils.huggingface import push_to_hub
|
351 |
+
|
352 |
+
repo_name = f"{args.env_id}-{args.exp_name}-seed{args.seed}"
|
353 |
+
repo_id = f"{args.hf_entity}/{repo_name}" if args.hf_entity else repo_name
|
354 |
+
push_to_hub(args, episodic_returns, repo_id, "DQN", f"runs/{run_name}", f"videos/{run_name}-eval")
|
355 |
+
|
356 |
+
envs.close()
|
357 |
+
writer.close()
|
events.out.tfevents.1685407342.ccaa1a6bd80b.5479.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1504c0f26a25a44e50955047d17d20bef99e87688e840af5bc736e5365832040
|
3 |
+
size 1309
|
poetry.lock
ADDED
The diff for this file is too large to render.
See raw diff
|
|
pyproject.toml
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
[tool.poetry]
|
2 |
+
name = "lagomorph"
|
3 |
+
version = "0.1.0"
|
4 |
+
description = "Fork of CleanRL focused on DQN training"
|
5 |
+
packages = [
|
6 |
+
{ include = "envs" },
|
7 |
+
{ include = "utils" },
|
8 |
+
]
|
9 |
+
keywords = ["reinforcement", "machine", "learning", "research"]
|
10 |
+
license="MIT"
|
11 |
+
readme = "README.md"
|
replay.mp4
ADDED
Binary file (380 kB). View file
|
|
videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-0.mp4
ADDED
Binary file (380 kB). View file
|
|
videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-1.mp4
ADDED
Binary file (339 kB). View file
|
|
videos/BreakoutNoFrameskip-v4__dqn_atari__1__1685407338-eval/rl-video-episode-8.mp4
ADDED
Binary file (380 kB). View file
|
|