Upload models

.gitignore

@@ -0,0 +1 @@
+*

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<pad>": 50257
+}

arguments.json

@@ -0,0 +1,69 @@
+{
+    "model_name_or_path": "/home/juntao/Projects/roo-dev-cty/models/proxy_model/gpt2-774m",
+    "max_length": 1024,
+    "trust_remote_code": true,
+    "train_datasets": [
+        [
+            "PrefOnlyRewardJSON01",
+            {
+                "proportion": 1.0,
+                "path": "/home/juntao/Projects/roo-dev-cty/data/roo/gold-generate-dataset-ultrafeedback/30k/train.json"
+            }
+        ]
+    ],
+    "eval_datasets": [
+        [
+            "PrefOnlyRewardJSON02",
+            {
+                "proportion": 1.0,
+                "path": "/home/juntao/Projects/roo-dev-cty/data/roo/gold-generate-dataset-ultrafeedback/30k/test.json"
+            }
+        ]
+    ],
+    "loss_type": "sequence-wise",
+    "epochs": 2,
+    "per_device_train_batch_size": 4,
+    "per_device_eval_batch_size": 4,
+    "gradient_accumulation_steps": 4,
+    "gradient_checkpointing": true,
+    "regularization": 0.001,
+    "normalize_score_during_training": false,
+    "normalizer_type": "ExponentialMovingAverage",
+    "normalizer_momentum": 0.9,
+    "lr": 2e-05,
+    "lr_scheduler_type": "cosine",
+    "lr_warmup_ratio": 0.03,
+    "weight_decay": 0.1,
+    "seed": 42,
+    "fp16": false,
+    "bf16": true,
+    "tf32": true,
+    "lm_coef": 0.01,
+    "freeze_transformer_layers": false,
+    "eval_strategy": "epoch",
+    "eval_interval": 1000000,
+    "need_eval": true,
+    "eval_split_ratio": null,
+    "output_dir": "/home/juntao/Projects/roo-dev-cty/experiments/outputs/score_lm/gpt2_774m_0910",
+    "log_type": "wandb",
+    "log_dir": "/home/juntao/Projects/roo-dev-cty/experiments/outputs/score_lm/gpt2_774m_0910",
+    "log_project": "score_lm",
+    "log_group": null,
+    "log_run_name": "gpt2_774m_0910",
+    "save_16bit": true,
+    "save_interval": 1000000,
+    "local_rank": 0,
+    "zero_stage": 3,
+    "offload": "none",
+    "deepspeed": false,
+    "deepspeed_config": null,
+    "deepscale": false,
+    "deepscale_config": null,
+    "global_rank": 0,
+    "device": {
+        "type": "torch.device",
+        "repr": "device(type='cuda', index=0)"
+    },
+    "num_update_steps_per_epoch": 211,
+    "total_training_steps": 422
+}

arguments.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8da817dc4351324ea8d981d9d4f700ed8bc9be54b9227394bf2ce64e3695da9f
+size 1555

config.json

@@ -0,0 +1,51 @@
+{
+  "_name_or_path": "/home/juntao/Projects/roo-dev-cty/models/proxy_model/gpt2-774m",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2ForScoreLM"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "do_normalize": false,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "mean": [
+    0.16215750575065613
+  ],
+  "model_type": "gpt2",
+  "momentum": 0.9,
+  "n_ctx": 1024,
+  "n_embd": 1280,
+  "n_head": 20,
+  "n_inner": null,
+  "n_layer": 36,
+  "n_positions": 1024,
+  "normalizer_type": "ExponentialMovingAverage",
+  "pad_token_id": 50257,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "score_bias": true,
+  "score_dim": 1,
+  "score_type": "reward",
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "transformers_version": "4.40.0",
+  "use_cache": true,
+  "var": [
+    4.382232189178467
+  ],
+  "vocab_size": 50258
+}

environ.txt

@@ -0,0 +1,118 @@
+BAT_THEME=Monokai Extended
+BROWSER=/home/juntao/.vscode-server/bin/4849ca9bdf9666755eb463db297b69e5385090e3/bin/helpers/browser.sh
+CC=/usr/bin/gcc
+COLORTERM=truecolor
+CONDA_DEFAULT_ENV=roo
+CONDA_EXE=/home/juntao/Miniconda3/bin/conda
+CONDA_PREFIX=/home/juntao/Miniconda3/envs/roo
+CONDA_PROMPT_MODIFIER=(roo) 
+CONDA_PYTHONBREAKPOINT=
+CONDA_PYTHON_EXE=/home/juntao/Miniconda3/bin/python
+CONDA_ROOT=/home/juntao/Miniconda3
+CONDA_SHLVL=1
+CPLUS_INCLUDE_PATH=/usr/local/cuda/include:/usr/local/cuda/extras/CUPTI/include
+CROSS_RANK=0
+CROSS_SIZE=1
+CUDA_HOME=/usr/local/cuda
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+CXX=/usr/bin/g++
+C_INCLUDE_PATH=/usr/local/cuda/include:/usr/local/cuda/extras/CUPTI/include
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1005/bus
+FC=/usr/bin/gfortran
+FPATH=/home/juntao/.local/share/gem/ruby/3.0.0/gems/colorls-1.4.6/zsh:/home/juntao/.oh-my-zsh/plugins/vscode:/home/juntao/.oh-my-zsh/plugins/brew:/home/juntao/.oh-my-zsh/plugins/tmux:/home/juntao/.oh-my-zsh/plugins/docker:/home/juntao/.oh-my-zsh/plugins/pylint:/home/juntao/.oh-my-zsh/plugins/pip:/home/juntao/.oh-my-zsh/plugins/python:/home/juntao/.oh-my-zsh/plugins/git-auto-fetch:/home/juntao/.oh-my-zsh/plugins/git:/home/juntao/.oh-my-zsh/plugins/alias-finder:/home/juntao/.oh-my-zsh/plugins/rsync:/home/juntao/.oh-my-zsh/plugins/cp:/home/juntao/.oh-my-zsh/plugins/copypath:/home/juntao/.oh-my-zsh/plugins/copyfile:/home/juntao/.oh-my-zsh/plugins/fzf:/home/juntao/.oh-my-zsh/plugins/colored-man-pages:/home/juntao/.oh-my-zsh/plugins/colorize:/home/juntao/.oh-my-zsh/custom/plugins/conda-zsh-completion:/home/juntao/.oh-my-zsh/custom/plugins/zsh-completions:/home/juntao/.oh-my-zsh/custom/plugins/zsh-autosuggestions:/home/juntao/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting:/home/juntao/.oh-my-zsh/plugins/ubuntu:/home/juntao/.oh-my-zsh/functions:/home/juntao/.oh-my-zsh/completions:/home/juntao/.oh-my-zsh/custom/functions:/home/juntao/.oh-my-zsh/custom/completions:/home/linuxbrew/.linuxbrew/share/zsh/site-functions:/home/juntao/.local/share/gem/ruby/3.0.0/gems/colorls-1.4.6/zsh:/home/juntao/.oh-my-zsh/plugins/vscode:/home/juntao/.oh-my-zsh/plugins/brew:/home/juntao/.oh-my-zsh/plugins/tmux:/home/juntao/.oh-my-zsh/plugins/docker:/home/juntao/.oh-my-zsh/plugins/pylint:/home/juntao/.oh-my-zsh/plugins/pip:/home/juntao/.oh-my-zsh/plugins/python:/home/juntao/.oh-my-zsh/plugins/git-auto-fetch:/home/juntao/.oh-my-zsh/plugins/git:/home/juntao/.oh-my-zsh/plugins/alias-finder:/home/juntao/.oh-my-zsh/plugins/rsync:/home/juntao/.oh-my-zsh/plugins/cp:/home/juntao/.oh-my-zsh/plugins/copypath:/home/juntao/.oh-my-zsh/plugins/copyfile:/home/juntao/.oh-my-zsh/plugins/fzf:/home/juntao/.oh-my-zsh/plugins/colored-man-pages:/home/juntao/.oh-my-zsh/plugins/colorize:/home/juntao/.oh-my-zsh/custom/plugins/conda-zsh-completion:/home/juntao/.oh-my-zsh/custom/plugins/zsh-completions:/home/juntao/.oh-my-zsh/custom/plugins/zsh-autosuggestions:/home/juntao/.oh-my-zsh/custom/plugins/zsh-syntax-highlighting:/home/juntao/.oh-my-zsh/plugins/ubuntu:/home/juntao/.oh-my-zsh/functions:/home/juntao/.oh-my-zsh/completions:/home/juntao/.oh-my-zsh/custom/functions:/home/juntao/.oh-my-zsh/custom/completions:/home/juntao/.oh-my-zsh/cache/completions:/usr/local/share/zsh/site-functions:/usr/share/zsh/vendor-functions:/usr/share/zsh/vendor-completions:/usr/share/zsh/functions/Calendar:/usr/share/zsh/functions/Chpwd:/usr/share/zsh/functions/Completion:/usr/share/zsh/functions/Completion/AIX:/usr/share/zsh/functions/Completion/BSD:/usr/share/zsh/functions/Completion/Base:/usr/share/zsh/functions/Completion/Cygwin:/usr/share/zsh/functions/Completion/Darwin:/usr/share/zsh/functions/Completion/Debian:/usr/share/zsh/functions/Completion/Linux:/usr/share/zsh/functions/Completion/Mandriva:/usr/share/zsh/functions/Completion/Redhat:/usr/share/zsh/functions/Completion/Solaris:/usr/share/zsh/functions/Completion/Unix:/usr/share/zsh/functions/Completion/X:/usr/share/zsh/functions/Completion/Zsh:/usr/share/zsh/functions/Completion/openSUSE:/usr/share/zsh/functions/Exceptions:/usr/share/zsh/functions/MIME:/usr/share/zsh/functions/Math:/usr/share/zsh/functions/Misc:/usr/share/zsh/functions/Newuser:/usr/share/zsh/functions/Prompts:/usr/share/zsh/functions/TCP:/usr/share/zsh/functions/VCS_Info:/usr/share/zsh/functions/VCS_Info/Backends:/usr/share/zsh/functions/Zftp:/usr/share/zsh/functions/Zle:/home/linuxbrew/.linuxbrew/share/zsh/functions:/home/juntao/.oh-my-zsh/custom/plugins/zsh-completions/src:/home/juntao/.oh-my-zsh/custom/plugins/zsh-completions/src:/home/juntao/.oh-my-zsh/custom/plugins/conda-zsh-completion:/home/linuxbrew/.linuxbrew/share/zsh/site-functions
+FZF_CTRL_T_COMMAND=fdfind --type file --follow --hidden --no-ignore-vcs --exclude '.git' --exclude '[Mm]iniconda3' --exclude '[Aa]naconda3' --color=always
+FZF_DEFAULT_COMMAND=fdfind --type file --follow --hidden --no-ignore-vcs --exclude '.git' --exclude '[Mm]iniconda3' --exclude '[Aa]naconda3' --color=always
+FZF_DEFAULT_OPTS=--height=40% --layout=reverse --ansi --preview='(batcat --color=always {} || highlight -O ansi {} || cat {}) 2>/dev/null | head -100'
+GIT_ASKPASS=/home/juntao/.vscode-server/bin/4849ca9bdf9666755eb463db297b69e5385090e3/extensions/git/dist/askpass.sh
+GSETTINGS_SCHEMA_DIR=/home/juntao/Miniconda3/envs/roo/share/glib-2.0/schemas
+GSETTINGS_SCHEMA_DIR_CONDA_BACKUP=
+HOME=/home/juntao
+HOMEBREW_API_DOMAIN=https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles/api
+HOMEBREW_BAT=true
+HOMEBREW_BOTTLE_DOMAIN=https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles
+HOMEBREW_BREW_GIT_REMOTE=https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git
+HOMEBREW_CELLAR=/home/linuxbrew/.linuxbrew/Cellar
+HOMEBREW_CORE_GIT_REMOTE=https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git
+HOMEBREW_EDITOR=vim
+HOMEBREW_PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
+HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew
+HOMEBREW_REPOSITORY=/home/linuxbrew/.linuxbrew/Homebrew
+INFOPATH=/home/linuxbrew/.linuxbrew/share/info:/home/linuxbrew/.linuxbrew/share/info:
+LANG=en_US.UTF-8
+LC_ADDRESS=zh_CN.UTF-8
+LC_ALL=en_US.utf8
+LC_IDENTIFICATION=zh_CN.UTF-8
+LC_MEASUREMENT=zh_CN.UTF-8
+LC_MONETARY=zh_CN.UTF-8
+LC_NAME=zh_CN.UTF-8
+LC_NUMERIC=zh_CN.UTF-8
+LC_PAPER=zh_CN.UTF-8
+LC_TELEPHONE=zh_CN.UTF-8
+LC_TIME=zh_CN.UTF-8
+LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+LESS=-R -M -i -j5
+LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
+LOCAL_RANK=0
+LOCAL_SIZE=8
+LOGLEVEL=DEBUG
+LOGNAME=juntao
+LSCOLORS=Gxfxcxdxbxegedabagacad
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.webp=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
+MASTER_ADDR=127.0.0.1
+MASTER_PORT=21326
+MOTD_SHOWN=pam
+MPICH_CC=/usr/bin/gcc
+MPICH_CXX=/usr/bin/g++
+MPICH_FC=/usr/bin/gfortran
+OLDPWD=/home/juntao/Projects/safe-rlhf
+OMPI_CC=/usr/bin/gcc
+OMPI_CXX=/usr/bin/g++
+OMPI_FC=/usr/bin/gfortran
+P9K_SSH=1
+P9K_TTY=old
+PAGER=less
+PATH=/home/juntao/Miniconda3/envs/roo/bin:/home/juntao/Miniconda3/envs/roo/bin:/home/juntao/.local/share/gem/ruby/3.0.0/bin:/var/lib/gems/3.0.0/bin:/usr/local/cuda/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin:/home/juntao/.local/bin:/home/juntao/.vscode-server/bin/4849ca9bdf9666755eb463db297b69e5385090e3/bin/remote-cli:/home/juntao/.perl/bin:/home/juntao/Miniconda3/condabin:/home/juntao/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/juntao/.fzf/bin
+PERL5LIB=/home/juntao/.perl/lib/perl5
+PERL_LOCAL_LIB_ROOT=/home/juntao/.perl
+PERL_MB_OPT=--install_base "/home/juntao/.perl"
+PERL_MM_OPT=INSTALL_BASE=/home/juntao/.perl
+PWD=/home/juntao/Projects/roo-dev-cty
+PYTHONBREAKPOINT=ipdb.set_trace
+PYTHONHASHSEED=42
+PYTHONPATH=/home/juntao/Projects/roo-dev-cty/experiments
+RANK=0
+RUBYOPT=-W0
+SHELL=/usr/bin/zsh
+SHLVL=3
+SSH_CLIENT=127.0.0.1 44828 10700
+SSH_CONNECTION=127.0.0.1 44828 127.0.0.1 10700
+TERM=xterm-256color
+TERM_PROGRAM=vscode
+TERM_PROGRAM_VERSION=1.93.0
+USER=juntao
+USER_ZDOTDIR=/home/juntao
+VSCODE_GIT_ASKPASS_EXTRA_ARGS=
+VSCODE_GIT_ASKPASS_MAIN=/home/juntao/.vscode-server/bin/4849ca9bdf9666755eb463db297b69e5385090e3/extensions/git/dist/askpass-main.js
+VSCODE_GIT_ASKPASS_NODE=/home/juntao/.vscode-server/bin/4849ca9bdf9666755eb463db297b69e5385090e3/node
+VSCODE_GIT_IPC_HANDLE=/run/user/1005/vscode-git-e07e4512e9.sock
+VSCODE_INJECTION=1
+VSCODE_IPC_HOOK_CLI=/run/user/1005/vscode-ipc-5cf7b2bd-b730-42c9-996a-1dc559ff1044.sock
+VSCODE_NONCE=c0cf5b90-c623-436e-be99-1454798faae5
+WANDB_SERVICE=2-3553582-tcp-localhost-58473
+WORLD_SIZE=8
+XDG_DATA_DIRS=/usr/share/gnome:/usr/local/share:/usr/share:/var/lib/snapd/desktop
+XDG_RUNTIME_DIR=/run/user/1005
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=7919
+XDG_SESSION_TYPE=tty
+ZDOTDIR=/home/juntao
+ZSH=/home/juntao/.oh-my-zsh
+ZSH_TMUX_CONFIG=/home/juntao/.tmux.conf
+ZSH_TMUX_TERM=screen-256color
+_=/home/juntao/Miniconda3/envs/roo/bin/deepspeed
+_CE_CONDA=
+_CE_M=
+_P9K_SSH_TTY=/dev/pts/73
+_P9K_TTY=/dev/pts/73
+_ZSH_TMUX_FIXED_CONFIG=/home/juntao/.oh-my-zsh/plugins/tmux/tmux.extra.conf

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b95072d783b8b8aa0415047ce69c2b23d9ef864993969116e3d888213c323cb
+size 1548203438

script.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+#
+# Copyright 2023-2024 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+if [ -z "${BASH_VERSION}" ]; then
+	echo "Please use bash to run this script." >&2
+	exit 1
+fi
+
+set -x
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" &>/dev/null && pwd)"
+ROOT_DIR="$(dirname "${SCRIPT_DIR}")"
+export PYTHONPATH="${ROOT_DIR}${PYTHONPATH:+:${PYTHONPATH}}"
+export LOGLEVEL="${LOGLEVEL:-DEBUG}"
+
+MODEL_LOCAL_DIR="/data/juntao/Models"
+MODEL_NAME_OR_PATH="TinyLlama/TinyLlama_v1.1"
+MODEL_NAME_TO_SAVE="gpt2_774m_0910"
+OUTPUT_DIR="${ROOT_DIR}/outputs/score_lm/${MODEL_NAME_TO_SAVE}"
+unset HOSTFILE
+ZERO_STAGE=3
+OFFLOAD="none"
+while [[ "$#" -gt 0 ]]; do
+	arg="$1"
+	shift
+	case "${arg}" in
+	--model_name_or_path)
+		MODEL_NAME_OR_PATH="$1"
+		shift
+		;;
+	--model_name_or_path=*)
+		MODEL_NAME_OR_PATH="${arg#*=}"
+		;;
+	--output_dir)
+		OUTPUT_DIR="$1"
+		shift
+		;;
+	--output_dir=*)
+		OUTPUT_DIR="${arg#*=}"
+		;;
+	--hostfile)
+		HOSTFILE="$1"
+		shift
+		;;
+	--hostfile=*)
+		HOSTFILE="${arg#*=}"
+		;;
+	--zero_stage)
+		ZERO_STAGE="$1"
+		shift
+		;;
+	--zero_stage=*)
+		ZERO_STAGE="${arg#*=}"
+		;;
+	--offload)
+		OFFLOAD="$1"
+		shift
+		;;
+	--offload=*)
+		OFFLOAD="${arg#*=}"
+		;;
+	*)
+		echo "Unknown parameter passed: '${arg}'" >&2
+		exit 1
+		;;
+	esac
+done
+
+MODEL_LOCAL_PATH="/home/juntao/Projects/roo-dev-cty/models/proxy_model/gpt2-774m"
+
+mkdir -p "${OUTPUT_DIR}"
+OUTPUT_DIR="$(cd "${OUTPUT_DIR}" &>/dev/null && pwd)"
+if [[ ! -f "${OUTPUT_DIR}/.gitignore" ]]; then
+	echo '*' >"${OUTPUT_DIR}/.gitignore"
+fi
+
+cp -f "$0" "${OUTPUT_DIR}/script.sh"
+
+WANDB_API_KEY="a198b7bd241afc7cc16f7fce0ef72122c21973c3"
+
+if [[ -z "${WANDB_API_KEY}" ]]; then
+	export WANDB_MODE="offline"
+fi
+
+MASTER_PORT_START=10000
+MASTER_PORT_END=65535
+MASTER_PORT="$(
+	comm -23 \
+		<(seq "${MASTER_PORT_START}" "${MASTER_PORT_END}" | sort) \
+		<(ss -Htan | awk '{ print $4 }' | awk -F ':' '{ print $NF }' | sort -u) |
+		shuf | head -n 1
+)"
+
+DEEPSPEED_ARGS=()
+if [[ -n "${HOSTFILE+x}" ]]; then
+	DEEPSPEED_ARGS+=("--hostfile" "${HOSTFILE}")
+fi
+DEEPSPEED_ARGS+=("--master_port" "${MASTER_PORT}")
+
+exec 1> >(tee "${OUTPUT_DIR}/stdout.log" >&1) 2> >(tee "${OUTPUT_DIR}/stderr.log" >&2)
+
+deepspeed "${DEEPSPEED_ARGS[@]}" \
+	--module safe_rlhf.values.score_lm \
+	--train_datasets PrefOnlyRewardJSON01::/home/juntao/Projects/roo-dev-cty/data/roo/gold-generate-dataset-ultrafeedback/30k/train.json \
+	--eval_datasets PrefOnlyRewardJSON02::/home/juntao/Projects/roo-dev-cty/data/roo/gold-generate-dataset-ultrafeedback/30k/test.json \
+	--model_name_or_path "${MODEL_LOCAL_PATH}" \
+	--max_length 1024 \
+	--trust_remote_code True \
+	--loss_type sequence-wise \
+	--epochs 2 \
+	--per_device_train_batch_size 4 \
+	--per_device_eval_batch_size 4 \
+	--gradient_accumulation_steps 4 \
+	--gradient_checkpointing \
+	--regularization 0.001 \
+	--normalize_score_during_training False \
+	--normalizer_type ExponentialMovingAverage \
+	--normalizer_momentum 0.9 \
+	--learning_rate 2e-5 \
+	--lr_scheduler_type cosine \
+	--lr_warmup_ratio 0.03 \
+	--weight_decay 0.1 \
+	--lm_coef 0.01 \
+	--seed 42 \
+	--need_eval \
+	--eval_strategy epoch \
+	--output_dir "${OUTPUT_DIR}" \
+	--log_type wandb \
+	--log_project score_lm \
+	--log_run_name "${MODEL_NAME_TO_SAVE}" \
+	--zero_stage "${ZERO_STAGE}" \
+	--offload "${OFFLOAD}" \
+	--bf16 True \
+	--tf32 True \
+	--save_16bit
+	# --freeze_transformer_layers \

special_tokens_map.json

@@ -0,0 +1,12 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}