+ source /admin/home/loubna/.bashrc ++ HISTCONTROL=ignoreboth ++ shopt -s histappend ++ HISTSIZE=1000 ++ HISTFILESIZE=2000 ++ shopt -s checkwinsize ++ '[' -x /usr/bin/lesspipe ']' +++ SHELL=/bin/sh +++ lesspipe ++ eval 'export LESSOPEN="| /usr/bin/lesspipe %s"; export LESSCLOSE="/usr/bin/lesspipe %s %s";' +++ export 'LESSOPEN=| /usr/bin/lesspipe %s' +++ LESSOPEN='| /usr/bin/lesspipe %s' +++ export 'LESSCLOSE=/usr/bin/lesspipe %s %s' +++ LESSCLOSE='/usr/bin/lesspipe %s %s' ++ '[' -z '' ']' ++ '[' -r /etc/debian_chroot ']' ++ case "$TERM" in ++ color_prompt=yes ++ '[' -n '' ']' ++ '[' yes = yes ']' ++ PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ unset color_prompt force_color_prompt ++ case "$TERM" in ++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ '[' -x /usr/bin/dircolors ']' ++ test -r /admin/home/loubna/.dircolors +++ dircolors -b ++ eval 'LS_COLORS='\''rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:'\''; export LS_COLORS' +++ LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:' +++ export LS_COLORS ++ alias 'ls=ls --color=auto' ++ alias 'grep=grep --color=auto' ++ alias 'fgrep=fgrep --color=auto' ++ alias 'egrep=egrep --color=auto' ++ alias 'll=ls -alF' ++ alias 'la=ls -A' ++ alias 'l=ls -CF' ++ alias 'alert=notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' ++ '[' -f /admin/home/loubna/.bash_aliases ']' ++ shopt -oq posix ++ '[' -f /usr/share/bash-completion/bash_completion ']' ++ . /usr/share/bash-completion/bash_completion +++ BASH_COMPLETION_VERSINFO=(2 10) +++ [[ ehxB == *v* ]] +++ BASH_COMPLETION_ORIGINAL_V_VALUE=+v +++ [[ -n '' ]] +++ set +v +++ _blacklist_glob='@(acroread.sh)' +++ shopt -s extglob progcomp +++ complete -u groups slay w sux +++ complete -A stopped -P '"%' -S '"' bg +++ complete -j -P '"%' -S '"' fg jobs disown +++ complete -v readonly unset +++ complete -A setopt set +++ complete -A shopt shopt +++ complete -A helptopic help +++ complete -a unalias +++ complete -c command type which +++ complete -b builtin +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ [[ linux-gnu == *@(solaris|aix)* ]] +++ _backup_glob='@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))' +++ complete -F _service service +++ _sysvdirs +++ sysvdirs=() +++ [[ -d /etc/rc.d/init.d ]] +++ [[ -d /etc/init.d ]] +++ sysvdirs+=(/etc/init.d) +++ [[ -f /etc/slackware-version ]] +++ return 0 +++ for svcdir in "${sysvdirs[@]}" +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/acpid ]] +++ complete -F _service /etc/init.d/acpid +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/apparmor ]] +++ complete -F _service /etc/init.d/apparmor +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/apport ]] +++ complete -F _service /etc/init.d/apport +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/atd ]] +++ complete -F _service /etc/init.d/atd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/chrony ]] +++ complete -F _service /etc/init.d/chrony +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/console-setup.sh ]] +++ complete -F _service /etc/init.d/console-setup.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cron ]] +++ complete -F _service /etc/init.d/cron +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cryptdisks ]] +++ complete -F _service /etc/init.d/cryptdisks +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/cryptdisks-early ]] +++ complete -F _service /etc/init.d/cryptdisks-early +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent ]] +++ complete -F _service /etc/init.d/datadog-agent +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-process ]] +++ complete -F _service /etc/init.d/datadog-agent-process +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-security ]] +++ complete -F _service /etc/init.d/datadog-agent-security +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/datadog-agent-trace ]] +++ complete -F _service /etc/init.d/datadog-agent-trace +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/dbus ]] +++ complete -F _service /etc/init.d/dbus +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/docker ]] +++ complete -F _service /etc/init.d/docker +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/gdrdrv ]] +++ complete -F _service /etc/init.d/gdrdrv +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/grub-common ]] +++ complete -F _service /etc/init.d/grub-common +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/hwclock.sh ]] +++ complete -F _service /etc/init.d/hwclock.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/irqbalance ]] +++ complete -F _service /etc/init.d/irqbalance +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/iscsid ]] +++ complete -F _service /etc/init.d/iscsid +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/iwpmd ]] +++ complete -F _service /etc/init.d/iwpmd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/keyboard-setup.sh ]] +++ complete -F _service /etc/init.d/keyboard-setup.sh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/kmod ]] +++ complete -F _service /etc/init.d/kmod +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/lvm2 ]] +++ complete -F _service /etc/init.d/lvm2 +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/lvm2-lvmpolld ]] +++ complete -F _service /etc/init.d/lvm2-lvmpolld +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/multipath-tools ]] +++ complete -F _service /etc/init.d/multipath-tools +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/munge ]] +++ complete -F _service /etc/init.d/munge +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/nfs-common ]] +++ complete -F _service /etc/init.d/nfs-common +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/open-iscsi ]] +++ complete -F _service /etc/init.d/open-iscsi +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/open-vm-tools ]] +++ complete -F _service /etc/init.d/open-vm-tools +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/plymouth ]] +++ complete -F _service /etc/init.d/plymouth +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/plymouth-log ]] +++ complete -F _service /etc/init.d/plymouth-log +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/procps ]] +++ complete -F _service /etc/init.d/procps +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rpcbind ]] +++ complete -F _service /etc/init.d/rpcbind +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rsync ]] +++ complete -F _service /etc/init.d/rsync +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/rsyslog ]] +++ complete -F _service /etc/init.d/rsyslog +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/screen-cleanup ]] +++ complete -F _service /etc/init.d/screen-cleanup +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/sendmail ]] +++ complete -F _service /etc/init.d/sendmail +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/ssh ]] +++ complete -F _service /etc/init.d/ssh +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/sysstat ]] +++ complete -F _service /etc/init.d/sysstat +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/udev ]] +++ complete -F _service /etc/init.d/udev +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/ufw ]] +++ complete -F _service /etc/init.d/ufw +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/unattended-upgrades ]] +++ complete -F _service /etc/init.d/unattended-upgrades +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/uuidd ]] +++ complete -F _service /etc/init.d/uuidd +++ for svc in $svcdir/!($_backup_glob) +++ [[ -x /etc/init.d/x11-common ]] +++ complete -F _service /etc/init.d/x11-common +++ unset svc svcdir sysvdirs +++ [[ linux-gnu == *freebsd* ]] +++ shopt -u hostcomplete +++ complete -F _user_at_host talk ytalk finger +++ complete -F _known_hosts traceroute traceroute6 fping fping6 telnet rsh rlogin ftp dig mtr ssh-installkeys showmount +++ shopt -q cdable_vars +++ complete -F _cd -o nospace cd pushd +++ complete -F _command aoss command do else eval exec ltrace nice nohup padsp then time tsocks vsound xargs +++ complete -F _root_command fakeroot gksu gksudo kdesudo really +++ complete -F _longopt a2ps awk base64 bash bc bison cat chroot colordiff cp csplit cut date df diff dir du enscript env expand fmt fold gperf grep grub head irb ld ldd less ln ls m4 md5sum mkdir mkfifo mknod mv netstat nl nm objcopy objdump od paste pr ptx readelf rm rmdir sed seq shasum sha1sum sha224sum sha256sum sha384sum sha512sum shar sort split strip sum tac tail tee texindex touch tr uname unexpand uniq units vdir wc who +++ [[ 5 -gt 4 ]] +++ declare -Ag _xspecs +++ _install_xspec '!*.?(t)bz?(2)' bunzip2 bzcat pbunzip2 pbzcat lbunzip2 lbzcat +++ local 'xspec=!*.?(t)bz?(2)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.?(t)bz?(2)' +++ _install_xspec '!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' unzip zipinfo +++ local 'xspec=!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|[egjswx]ar|exe|pk3|wsz|zargo|xpi|s[tx][cdiw]|sx[gm]|o[dt][tspgfc]|od[bm]|oxt|epub|apk|aab|ipa|do[ct][xm]|p[op]t[mx]|xl[st][xm]|pyz|whl)' +++ _install_xspec '*.Z' compress znew +++ local 'xspec=*.Z' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='*.Z' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.Z' +++ _install_xspec '!*.@(Z|[gGd]z|t[ag]z)' gunzip zcat +++ local 'xspec=!*.@(Z|[gGd]z|t[ag]z)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGd]z|t[ag]z)' +++ _install_xspec '!*.@(Z|[gGdz]z|t[ag]z)' unpigz +++ local 'xspec=!*.@(Z|[gGdz]z|t[ag]z)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(Z|[gGdz]z|t[ag]z)' +++ _install_xspec '!*.Z' uncompress +++ local 'xspec=!*.Z' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.Z' +++ _install_xspec '!*.@(tlz|lzma)' lzcat lzegrep lzfgrep lzgrep lzless lzmore unlzma +++ local 'xspec=!*.@(tlz|lzma)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(tlz|lzma)' +++ _install_xspec '!*.@(?(t)xz|tlz|lzma)' unxz xzcat +++ local 'xspec=!*.@(?(t)xz|tlz|lzma)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(t)xz|tlz|lzma)' +++ _install_xspec '!*.lrz' lrunzip +++ local 'xspec=!*.lrz' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.lrz' +++ _install_xspec '!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' ee +++ local 'xspec=!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx)' +++ _install_xspec '!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' qiv +++ local 'xspec=!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|svg)' +++ _install_xspec '!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' xv +++ local 'xspec=!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(gif|jp?(e)g?(2)|j2[ck]|jp[2f]|tif?(f)|png|p[bgp]m|bmp|x[bp]m|rle|rgb|pcx|fits|pm|?(e)ps)' +++ _install_xspec '!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' gv ggv kghostview +++ local 'xspec=!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(@(?(e)ps|?(E)PS|pdf|PDF)?(.gz|.GZ|.bz2|.BZ2|.Z))' +++ _install_xspec '!*.@(dvi|DVI)?(.@(gz|Z|bz2))' xdvi kdvi +++ local 'xspec=!*.@(dvi|DVI)?(.@(gz|Z|bz2))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dvi|DVI)?(.@(gz|Z|bz2))' +++ _install_xspec '!*.dvi' dvips dviselect dvitype dvipdf advi dvipdfm dvipdfmx +++ local 'xspec=!*.dvi' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.dvi' +++ _install_xspec '!*.[pf]df' acroread gpdf +++ local 'xspec=!*.[pf]df' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[pf]df' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[pf]df' +++ _install_xspec '!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' xpdf +++ local 'xspec=!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(pdf|fdf)?(.@(gz|GZ|bz2|BZ2|Z))' +++ _install_xspec '!*.@(?(e)ps|pdf)' kpdf +++ local 'xspec=!*.@(?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ _install_xspec '!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' okular +++ local 'xspec=!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(okular|@(?(e|x)ps|?(E|X)PS|[pf]df|[PF]DF|dvi|DVI|cb[rz]|CB[RZ]|djv?(u)|DJV?(U)|dvi|DVI|gif|jp?(e)g|miff|tif?(f)|pn[gm]|p[bgp]m|bmp|xpm|ico|xwd|tga|pcx|GIF|JP?(E)G|MIFF|TIF?(F)|PN[GM]|P[BGP]M|BMP|XPM|ICO|XWD|TGA|PCX|epub|EPUB|odt|ODT|fb?(2)|FB?(2)|mobi|MOBI|g3|G3|chm|CHM)?(.?(gz|GZ|bz2|BZ2|xz|XZ)))' +++ _install_xspec '!*.pdf' epdfview pdfunite +++ local 'xspec=!*.pdf' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.pdf' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.pdf' +++ _install_xspec '!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' zathura +++ local 'xspec=!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(cb[rz7t]|djv?(u)|?(e)ps|pdf)' +++ _install_xspec '!*.@(?(e)ps|pdf)' ps2pdf ps2pdf12 ps2pdf13 ps2pdf14 ps2pdfwr +++ local 'xspec=!*.@(?(e)ps|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(e)ps|pdf)' +++ _install_xspec '!*.texi*' makeinfo texi2html +++ local 'xspec=!*.texi*' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.texi*' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.texi*' +++ _install_xspec '!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' tex latex slitex jadetex pdfjadetex pdftex pdflatex texi2dvi xetex xelatex luatex lualatex +++ local 'xspec=!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?(la)tex|texi|dtx|ins|ltx|dbj)' +++ _install_xspec '!*.mp3' mpg123 mpg321 madplay +++ local 'xspec=!*.mp3' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.mp3' +++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' xine aaxine fbxine +++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ _install_xspec '!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' kaffeine dragon +++ local 'xspec=!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*@(.@(mp?(e)g|MP?(E)G|wm[av]|WM[AV]|avi|AVI|asf|vob|VOB|bin|dat|divx|DIVX|vcd|ps|pes|fli|flv|FLV|fxm|FXM|viv|rm|ram|yuv|mov|MOV|qt|QT|web[am]|WEB[AM]|mp[234]|MP[234]|m?(p)4[av]|M?(P)4[AV]|mkv|MKV|og[agmvx]|OG[AGMVX]|t[ps]|T[PS]|m2t?(s)|M2T?(S)|mts|MTS|wav|WAV|flac|FLAC|asx|ASX|mng|MNG|srt|m[eo]d|M[EO]D|s[3t]m|S[3T]M|it|IT|xm|XM|iso|ISO)|+([0-9]).@(vdr|VDR))?(.@(crdownload|part))' +++ _install_xspec '!*.@(avi|asf|wmv)' aviplay +++ local 'xspec=!*.@(avi|asf|wmv)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(avi|asf|wmv)' +++ _install_xspec '!*.@(rm?(j)|ra?(m)|smi?(l))' realplay +++ local 'xspec=!*.@(rm?(j)|ra?(m)|smi?(l))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(rm?(j)|ra?(m)|smi?(l))' +++ _install_xspec '!*.@(mpg|mpeg|avi|mov|qt)' xanim +++ local 'xspec=!*.@(mpg|mpeg|avi|mov|qt)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mpg|mpeg|avi|mov|qt)' +++ _install_xspec '!*.@(og[ag]|m3u|flac|spx)' ogg123 +++ local 'xspec=!*.@(og[ag]|m3u|flac|spx)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(og[ag]|m3u|flac|spx)' +++ _install_xspec '!*.@(mp3|og[ag]|pls|m3u)' gqmpeg freeamp +++ local 'xspec=!*.@(mp3|og[ag]|pls|m3u)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp3|og[ag]|pls|m3u)' +++ _install_xspec '!*.fig' xfig +++ local 'xspec=!*.fig' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.fig' +++ _install_xspec '!*.@(mid?(i)|cmf)' playmidi +++ local 'xspec=!*.@(mid?(i)|cmf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mid?(i)|cmf)' +++ _install_xspec '!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' timidity +++ local 'xspec=!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mid?(i)|rmi|rcp|[gr]36|g18|mod|xm|it|x3m|s[3t]m|kar)' +++ _install_xspec '!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' modplugplay modplug123 +++ local 'xspec=!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(669|abc|am[fs]|d[bs]m|dmf|far|it|mdl|m[eo]d|mid?(i)|mt[2m]|oct|okt?(a)|p[st]m|s[3t]m|ult|umx|wav|xm)' +++ _install_xspec '*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' vi vim gvim rvim view rview rgvim rgview gview emacs xemacs sxemacs kate kwrite +++ local 'xspec=*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ for cmd in "$@" +++ _xspecs[$cmd]='*.@([ao]|so|so.!(conf|*/*)|[rs]pm|gif|jp?(e)g|mp3|mp?(e)g|avi|asf|ogg|class)' +++ _install_xspec '!*.@(zip|z|gz|tgz)' bzme +++ local 'xspec=!*.@(zip|z|gz|tgz)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(zip|z|gz|tgz)' +++ _install_xspec '!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' netscape mozilla lynx galeon dillo elinks amaya epiphany +++ local 'xspec=!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(?([xX]|[sS])[hH][tT][mM]?([lL]))' +++ _install_xspec '!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' oowriter lowriter +++ local 'xspec=!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxw|stw|sxg|sgl|doc?([mx])|dot?([mx])|rtf|txt|htm|html|?(f)odt|ott|odm|pdf)' +++ _install_xspec '!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' ooimpress loimpress +++ local 'xspec=!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxi|sti|pps?(x)|ppt?([mx])|pot?([mx])|?(f)odp|otp)' +++ _install_xspec '!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' oocalc localc +++ local 'xspec=!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxc|stc|xls?([bmx])|xlw|xlt?([mx])|[ct]sv|?(f)ods|ots)' +++ _install_xspec '!*.@(sxd|std|sda|sdd|?(f)odg|otg)' oodraw lodraw +++ local 'xspec=!*.@(sxd|std|sda|sdd|?(f)odg|otg)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxd|std|sda|sdd|?(f)odg|otg)' +++ _install_xspec '!*.@(sxm|smf|mml|odf)' oomath lomath +++ local 'xspec=!*.@(sxm|smf|mml|odf)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(sxm|smf|mml|odf)' +++ _install_xspec '!*.odb' oobase lobase +++ local 'xspec=!*.odb' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.odb' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.odb' +++ _install_xspec '!*.[rs]pm' rpm2cpio +++ local 'xspec=!*.[rs]pm' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[rs]pm' +++ _install_xspec '!*.aux' bibtex +++ local 'xspec=!*.aux' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.aux' +++ _install_xspec '!*.po' poedit gtranslator kbabel lokalize +++ local 'xspec=!*.po' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.po' +++ _install_xspec '!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' harbour gharbour hbpp +++ local 'xspec=!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@([Pp][Rr][Gg]|[Cc][Ll][Pp])' +++ _install_xspec '!*.[Hh][Rr][Bb]' hbrun +++ local 'xspec=!*.[Hh][Rr][Bb]' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.[Hh][Rr][Bb]' +++ _install_xspec '!*.ly' lilypond ly2dvi +++ local 'xspec=!*.ly' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.ly' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.ly' +++ _install_xspec '!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cdiff +++ local 'xspec=!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(dif?(f)|?(d)patch)?(.@([gx]z|bz2|lzma))' +++ _install_xspec '!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' portecle +++ local 'xspec=!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!@(*.@(ks|jks|jceks|p12|pfx|bks|ubr|gkr|cer|crt|cert|p7b|pkipath|pem|p10|csr|crl)|cacerts)' +++ _install_xspec '!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' kid3 kid3-qt +++ local 'xspec=!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' cmd +++ shift +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' +++ for cmd in "$@" +++ _xspecs[$cmd]='!*.@(mp[234c]|og[ag]|@(fl|a)ac|m4[abp]|spx|tta|w?(a)v|wma|aif?(f)|asf|ape)' +++ unset -f _install_xspec +++ complete -F _minimal '' +++ complete -D -F _completion_loader +++ compat_dir=/etc/bash_completion.d +++ [[ -d /etc/bash_completion.d ]] +++ [[ -r /etc/bash_completion.d ]] +++ [[ -x /etc/bash_completion.d ]] +++ for i in "$compat_dir"/* +++ [[ apport_completion != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] +++ [[ -f /etc/bash_completion.d/apport_completion ]] +++ [[ -r /etc/bash_completion.d/apport_completion ]] +++ . /etc/bash_completion.d/apport_completion ++++ complete -F _apport-bug -o filenames -o dirnames ubuntu-bug ++++ complete -F _apport-bug -o filenames -o dirnames apport-bug ++++ complete -F _apport-cli -o filenames -o dirnames apport-cli ++++ complete -F _apport-unpack -o filenames -o dirnames apport-unpack ++++ complete -F _apport-collect apport-collect +++ for i in "$compat_dir"/* +++ [[ git-prompt != @(@(#*#|*@(~|.@(bak|orig|rej|swp|dpkg*|rpm@(orig|new|save))))|Makefile*|@(acroread.sh)) ]] +++ [[ -f /etc/bash_completion.d/git-prompt ]] +++ [[ -r /etc/bash_completion.d/git-prompt ]] +++ . /etc/bash_completion.d/git-prompt ++++ [[ -e /usr/lib/git-core/git-sh-prompt ]] ++++ . /usr/lib/git-core/git-sh-prompt +++++ __git_printf_supports_v= +++++ printf -v __git_printf_supports_v -- %s yes +++ unset compat_dir i _blacklist_glob +++ user_completion=/admin/home/loubna/.bash_completion +++ [[ /usr/share/bash-completion/bash_completion != /admin/home/loubna/.bash_completion ]] +++ [[ -r /admin/home/loubna/.bash_completion ]] +++ unset user_completion +++ unset -f have +++ unset have +++ set +v +++ unset BASH_COMPLETION_ORIGINAL_V_VALUE ++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin +++ /fsx/loubna/miniconda3/bin/conda shell.bash hook ++ __conda_setup='export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause __conda_exe() ( "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" ) __conda_hashr() { if [ -n "${ZSH_VERSION:+x}" ]; then \rehash elif [ -n "${POSH_VERSION:+x}" ]; then : # pass else \hash -r fi } __conda_activate() { if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then # Handle transition from shell activated with conda <= 4.3 to a subsequent activation # after conda updated to >= 4.4. See issue #6173. PS1="$CONDA_PS1_BACKUP" \unset CONDA_PS1_BACKUP fi \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return \eval "$ask_conda" __conda_hashr } __conda_reactivate() { \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return \eval "$ask_conda" __conda_hashr } conda() { \local cmd="${1-__missing__}" case "$cmd" in activate|deactivate) __conda_activate "$@" ;; install|update|upgrade|remove|uninstall) __conda_exe "$@" || \return __conda_reactivate ;; *) __conda_exe "$@" ;; esac } if [ -z "${CONDA_SHLVL+x}" ]; then \export CONDA_SHLVL=0 # In dev-mode CONDA_EXE is python.exe and on Windows # it is in a different relative location to condabin. if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" else PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" fi \export PATH # We'\''re not allowing PS1 to be unbound. It must at least be set. # However, we'\''re not exporting it, which can cause problems when starting a second shell # via a first shell (i.e. starting zsh from bash). if [ -z "${PS1+x}" ]; then PS1= fi fi conda activate base' ++ '[' 0 -eq 0 ']' ++ eval 'export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\'' # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause __conda_exe() ( "$CONDA_EXE" $_CE_M $_CE_CONDA "$@" ) __conda_hashr() { if [ -n "${ZSH_VERSION:+x}" ]; then \rehash elif [ -n "${POSH_VERSION:+x}" ]; then : # pass else \hash -r fi } __conda_activate() { if [ -n "${CONDA_PS1_BACKUP:+x}" ]; then # Handle transition from shell activated with conda <= 4.3 to a subsequent activation # after conda updated to >= 4.4. See issue #6173. PS1="$CONDA_PS1_BACKUP" \unset CONDA_PS1_BACKUP fi \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix "$@")" || \return \eval "$ask_conda" __conda_hashr } __conda_reactivate() { \local ask_conda ask_conda="$(PS1="${PS1:-}" __conda_exe shell.posix reactivate)" || \return \eval "$ask_conda" __conda_hashr } conda() { \local cmd="${1-__missing__}" case "$cmd" in activate|deactivate) __conda_activate "$@" ;; install|update|upgrade|remove|uninstall) __conda_exe "$@" || \return __conda_reactivate ;; *) __conda_exe "$@" ;; esac } if [ -z "${CONDA_SHLVL+x}" ]; then \export CONDA_SHLVL=0 # In dev-mode CONDA_EXE is python.exe and on Windows # it is in a different relative location to condabin. if [ -n "${_CE_CONDA:+x}" ] && [ -n "${WINDIR+x}" ]; then PATH="$(\dirname "$CONDA_EXE")/condabin${PATH:+":${PATH}"}" else PATH="$(\dirname "$(\dirname "$CONDA_EXE")")/condabin${PATH:+":${PATH}"}" fi \export PATH # We'\''re not allowing PS1 to be unbound. It must at least be set. # However, we'\''re not exporting it, which can cause problems when starting a second shell # via a first shell (i.e. starting zsh from bash). if [ -z "${PS1+x}" ]; then PS1= fi fi conda activate base' +++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda +++ export _CE_M= +++ _CE_M= +++ export _CE_CONDA= +++ _CE_CONDA= +++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ '[' -z x ']' +++ conda activate base +++ local cmd=activate +++ case "$cmd" in +++ __conda_activate activate base +++ '[' -n '' ']' +++ local ask_conda ++++ PS1='\[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++++ __conda_exe shell.posix activate base ++++ /fsx/loubna/miniconda3/bin/conda shell.posix activate base +++ ask_conda='PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' export CONDA_SHLVL='\''3'\'' export CONDA_DEFAULT_ENV='\''base'\'' export CONDA_PROMPT_MODIFIER='\''(base) '\'' export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/megatron'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' +++ eval 'PS1='\''(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3'\'' export CONDA_SHLVL='\''3'\'' export CONDA_DEFAULT_ENV='\''base'\'' export CONDA_PROMPT_MODIFIER='\''(base) '\'' export CONDA_PREFIX_2='\''/fsx/loubna/miniconda3/envs/megatron'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++++ export PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++++ PATH=/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++++ export CONDA_PREFIX=/fsx/loubna/miniconda3 ++++ CONDA_PREFIX=/fsx/loubna/miniconda3 ++++ export CONDA_SHLVL=3 ++++ CONDA_SHLVL=3 ++++ export CONDA_DEFAULT_ENV=base ++++ CONDA_DEFAULT_ENV=base ++++ export 'CONDA_PROMPT_MODIFIER=(base) ' ++++ CONDA_PROMPT_MODIFIER='(base) ' ++++ export CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/megatron ++++ CONDA_PREFIX_2=/fsx/loubna/miniconda3/envs/megatron ++++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++++ export _CE_M= ++++ _CE_M= ++++ export _CE_CONDA= ++++ _CE_CONDA= ++++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python +++ __conda_hashr +++ '[' -n '' ']' +++ '[' -n '' ']' +++ hash -r ++ unset __conda_setup ++ export WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb ++ WANDB_CACHE_DIR=/fsx/loubna/.tmp/wandb ++ export TMPDIR=/fsx/loubna/.tmp ++ TMPDIR=/fsx/loubna/.tmp ++ export HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache ++ HUGGINGFACE_HUB_CACHE=/fsx/loubna/.cache ++ export HF_DATASETS_CACHE=/fsx/loubna/.cache ++ HF_DATASETS_CACHE=/fsx/loubna/.cache ++ export PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin ++ export LD_LIBRARY_PATH=/usr/local/cuda-11.6/lib64:/usr/local/cuda-11.6/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib: ++ LD_LIBRARY_PATH=/usr/local/cuda-11.6/lib64:/usr/local/cuda-11.6/lib64:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib:/opt/amazon/efa/lib:/opt/amazon/openmpi/lib:/usr/local/cuda/efa/lib:/usr/local/lib:/usr/lib: ++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin ++ '[' -f /fsx/loubna/google-cloud-sdk/path.bash.inc ']' ++ . /fsx/loubna/google-cloud-sdk/path.bash.inc ++++ command readlink /fsx/loubna/google-cloud-sdk/path.bash.inc ++++ readlink /fsx/loubna/google-cloud-sdk/path.bash.inc +++ script_link= +++ script_link=/fsx/loubna/google-cloud-sdk/path.bash.inc +++ apparent_sdk_dir=/fsx/loubna/google-cloud-sdk +++ '[' /fsx/loubna/google-cloud-sdk == /fsx/loubna/google-cloud-sdk/path.bash.inc ']' ++++ command cd -P /fsx/loubna/google-cloud-sdk ++++ cd -P /fsx/loubna/google-cloud-sdk ++++ command pwd -P ++++ pwd -P +++ sdk_dir=/fsx/loubna/google-cloud-sdk +++ bin_path=/fsx/loubna/google-cloud-sdk/bin +++ [[ :/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin: != *\:\/\f\s\x\/\l\o\u\b\n\a\/\g\o\o\g\l\e\-\c\l\o\u\d\-\s\d\k\/\b\i\n\:* ]] ++ '[' -f /fsx/loubna/google-cloud-sdk/completion.bash.inc ']' ++ . /fsx/loubna/google-cloud-sdk/completion.bash.inc +++ complete -o nospace -F _python_argcomplete gcloud +++ unset bq_COMMANDS +++ complete -F _bq_completer bq +++ complete -o nospace -F _python_argcomplete gsutil + conda activate megatron + local cmd=activate + case "$cmd" in + __conda_activate activate megatron + '[' -n '' ']' + local ask_conda ++ PS1='(base) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ __conda_exe shell.posix activate megatron ++ /fsx/loubna/miniconda3/bin/conda shell.posix activate megatron + ask_conda='PS1='\''(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/megatron'\'' export CONDA_SHLVL='\''4'\'' export CONDA_DEFAULT_ENV='\''megatron'\'' export CONDA_PROMPT_MODIFIER='\''(megatron) '\'' export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' + eval 'PS1='\''(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '\'' export PATH='\''/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin'\'' export CONDA_PREFIX='\''/fsx/loubna/miniconda3/envs/megatron'\'' export CONDA_SHLVL='\''4'\'' export CONDA_DEFAULT_ENV='\''megatron'\'' export CONDA_PROMPT_MODIFIER='\''(megatron) '\'' export CONDA_PREFIX_3='\''/fsx/loubna/miniconda3'\'' export CONDA_EXE='\''/fsx/loubna/miniconda3/bin/conda'\'' export _CE_M='\'''\'' export _CE_CONDA='\'''\'' export CONDA_PYTHON_EXE='\''/fsx/loubna/miniconda3/bin/python'\''' ++ PS1='(megatron) \[\e]0;\u@\h: \w\a\]${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' ++ export PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin ++ PATH=/usr/local/cuda-11.6/bin:/opt/slurm/bin:/opt/slurm/sbin:/fsx/loubna/google-cloud-sdk/bin:/usr/local/cuda-11.6/bin:/fsx/loubna/miniconda3/envs/megatron/bin:/fsx/loubna/miniconda3/condabin:/opt/slurm/bin:/opt/slurm/sbin:/admin/home/loubna/.vscode-server/bin/d045a5eda657f4d7b676dedbfa7aab8207f8a075/bin/remote-cli:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/opt/amazon/openmpi/bin:/opt/amazon/efa/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/admin/home/loubna/.local/bin:/admin/home/loubna/.local/bin ++ export CONDA_PREFIX=/fsx/loubna/miniconda3/envs/megatron ++ CONDA_PREFIX=/fsx/loubna/miniconda3/envs/megatron ++ export CONDA_SHLVL=4 ++ CONDA_SHLVL=4 ++ export CONDA_DEFAULT_ENV=megatron ++ CONDA_DEFAULT_ENV=megatron ++ export 'CONDA_PROMPT_MODIFIER=(megatron) ' ++ CONDA_PROMPT_MODIFIER='(megatron) ' ++ export CONDA_PREFIX_3=/fsx/loubna/miniconda3 ++ CONDA_PREFIX_3=/fsx/loubna/miniconda3 ++ export CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++ CONDA_EXE=/fsx/loubna/miniconda3/bin/conda ++ export _CE_M= ++ _CE_M= ++ export _CE_CONDA= ++ _CE_CONDA= ++ export CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python ++ CONDA_PYTHON_EXE=/fsx/loubna/miniconda3/bin/python + __conda_hashr + '[' -n '' ']' + '[' -n '' ']' + hash -r ++ date + echo 'START TIME: Thu Jun 15 12:30:28 UTC 2023' START TIME: Thu Jun 15 12:30:28 UTC 2023 + BRRR_REPO=/fsx/loubna/code/fork/brrr + SCRIPT_REPO=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa + pushd /fsx/loubna/code/fork/brrr/examples/gpt2_mqa /fsx/loubna/code/fork/brrr/examples/gpt2_mqa /fsx/loubna/code/brrr/starcoder-5b-noconf + GPUS_PER_NODE=8 + NNODES=16 ++ head -n 1 ++ scontrol show hostnames 'ip-26-0-147-[141,187,189,193,204,233,245,247],ip-26-0-148-[55,93,115,151,170,193,245],ip-26-0-149-1' + MASTER_ADDR=ip-26-0-147-141 + MASTER_PORT=6000 + CONFIG_FILE=/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + export USE_FAST=1 + USE_FAST=1 + export CUDA_DEVICE_MAX_CONNECTIONS=1 + CUDA_DEVICE_MAX_CONNECTIONS=1 + CMD=' main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ' + export 'LAUNCHER=python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' + LAUNCHER='python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 ' + echo main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml + export NCCL_ASYNC_ERROR_HANDLING=1 + NCCL_ASYNC_ERROR_HANDLING=1 + export NCCL_PROTO=simple + NCCL_PROTO=simple + export RDMAV_FORK_SAFE=1 + RDMAV_FORK_SAFE=1 + export FI_EFA_FORK_SAFE=1 + FI_EFA_FORK_SAFE=1 + export FI_EFA_USE_DEVICE_RDMA=1 + FI_EFA_USE_DEVICE_RDMA=1 + export FI_PROVIDER=efa + FI_PROVIDER=efa + export FI_LOG_LEVEL=1 + FI_LOG_LEVEL=1 + export NCCL_IB_DISABLE=1 + NCCL_IB_DISABLE=1 + export NCCL_SOCKET_IFNAME=ens + NCCL_SOCKET_IFNAME=ens + SRUN_ARGS=' --wait=60 --kill-on-bad-exit=1 ' + SLURM_JOB_ID=158050 + srun --wait=60 --kill-on-bad-exit=1 --jobid 158050 -u bash -c 'python -u -m torch.distributed.run --nproc_per_node 8 --nnodes 16 --rdzv_endpoint ip-26-0-147-141:6000 --rdzv_backend c10d --max_restarts 0 --tee 3 --node_rank $SLURM_PROCID --role $SLURMD_NODENAME: main.py --config-file /fsx/loubna/code/fork/brrr/examples/gpt2_mqa/configs/config_5b_explicit_model_conf.yaml ' WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** WARNING:__main__: ***************************************** Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. ***************************************** [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: We use yaml config for model architecture [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: Config(general=GeneralArgs(name='brrr_example_gpt2_mqa_starcoder_5b_2', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: kill_switch_path=PosixPath('/fsx/loubna/br4-experiments/kill_loubna_starcoder'), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: ignore_sanity_checks=True), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: profile=None, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: checkpoints=CheckpointsArgs(checkpoints_path=PosixPath('/fsx/loubna/br4-experiments/checkpoints'), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: checkpoint_interval=2500, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: load_from_specific_checkpoint=None, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: upload_s3_path=None, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: upload_s3_num_workers=None), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: parallelism=ParallelismArgs(dp=32, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: pp=1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: tp=4, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: pp_engine=, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: tp_mode=, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: recompute_granularity=None, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: tp_column_linear_async_communication=True), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: model=ModelArgs(hf_gpt2_model_name='loubnabnl/starcoder-5b-noconf', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: hidden_size=3584, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: num_attention_heads=28, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: n_inner=14336, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: n_layer=40, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: max_position_embeddings=8192, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: vocab_size=49152, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: layer_norm_epsilon=1e-05, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: scale_attn_weights=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: activation_function='gelu', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: attention_softmax_in_fp32=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: resid_pdrop=0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: attn_pdrop=0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: embd_pdrop=0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: initializer_range=0.02, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: pad_key_length=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: scale_attention_softmax_in_fp32=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: make_vocab_size_divisible_by=128, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: dtype=torch.bfloat16, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: init_method=RandomInit(std=0.01275), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: seed=42), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: logging=LoggingArgs(log_level='info', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: log_level_replica='info', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: iteration_step_info_interval=1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: tensorboard_logger=TensorboardLoggerConfig(tensorboard_dir=PosixPath('/fsx/loubna/br4-experiments/tensorboard'))), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: tokens=TokensArgs(sequence_length=8192, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: train_steps=29652, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: micro_batch_size=2, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: batch_accumulation_per_replica=8, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: val_check_interval=2500, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: limit_val_batches=2, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: limit_test_batches=0), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: optimizer=OptimizerArgs(zero_stage=1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: weight_decay=0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: clip_grad=1.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: accumulate_grad_in_fp32=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: adam_eps=1e-08, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: adam_beta1=0.9, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: adam_beta2=0.95, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: learning_rate=0.0003), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: learning_rate_scheduler=LRSchedulerArgs(lr_warmup_steps=2000, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: lr_warmup_style='linear', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: lr_decay_style='cosine', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: lr_decay_steps=29652, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: min_decay_lr=3e-05), [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: data=DataArgs(seed=1234, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: num_loading_workers=2, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: dataset=PretrainNemoArgs(data_prefix=[3.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 53.89, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.78, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.85, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 5.68, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.31, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.98, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.08, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.09, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.12, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 23.78, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.7, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.61, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.26, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.68, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 2.23, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.3, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.31, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.45, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.12, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 6.81, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 9.11, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.06, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 44.66, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.58, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 2.23, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.25, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.03, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.31, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 2.87, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 3.32, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.19, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.39, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 5.2, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.02, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.56, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.07, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.41, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 3.66, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.56, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.03, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.001, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.23, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.02, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 4.69, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.35, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.33, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 3.09, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.46, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.2, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.04, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 11.09, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.4, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.3, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.42, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 48.92, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.64, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.4, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.71, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.91, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 29.36, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 86.94, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 64.71, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 74.93, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 60.89, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 60.4, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 26.52, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.001, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.42, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.94, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.01, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.0002, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.11, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.18, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 0.05, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 1.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 54.4, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 32.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 7.12, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: 6.0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: '/fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document'], [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: index_mapping_dir=None, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: splits_string='0.969,0.999,1', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: skip_warmup=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: dataloader_type='single', [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: validation_drop_last=True, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: eod_mask_loss=False, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: no_seqlen_plus_one_input_tokens=False, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: pad_samples_to_global_batch_size=False))) [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: GPTBigCodeConfig { [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "activation_function": "gelu", [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "attention_softmax_in_fp32": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "attention_type": 1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "attn_pdrop": 0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "bos_token_id": 50256, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "embd_pdrop": 0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "eos_token_id": 50256, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "hf_gpt2_model_name": "loubnabnl/starcoder-5b-noconf", [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "inference_runner": 0, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "initializer_range": 0.02, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "layer_norm_epsilon": 1e-05, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "make_vocab_size_divisible_by": 128, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "max_batch_size": null, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "max_sequence_length": null, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "model_type": "gpt_bigcode", [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "n_embd": 3584, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "n_head": 28, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "n_inner": 14336, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "n_layer": 40, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "n_positions": 8192, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "pad_key_length": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "pre_allocate_kv_cache": false, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "resid_pdrop": 0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "scale_attention_softmax_in_fp32": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "scale_attn_weights": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "seed": 42, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "summary_activation": null, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "summary_first_dropout": 0.1, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "summary_proj_to_labels": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "summary_type": "cls_index", [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "summary_use_proj": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "transformers_version": "4.27.0.dev0", [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "use_cache": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "validate_runner_input": true, [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: "vocab_size": 49152 [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: } [ip-26-0-147-141:0]:06/15/2023 12:30:41 [INFO|DP=0|PP=0|TP=0]: [ip-26-0-147-141:2]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=2]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) [ip-26-0-147-141:2]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=2]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB [ip-26-0-147-141:3]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=3]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) [ip-26-0-147-141:3]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=3]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB [ip-26-0-147-141:1]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=1]: Number of parameters: 1373356032 (2619.47MB). Expecting peak 4*param_size=10477.88MB with grads and Adam optim states (w/o memory optims) [ip-26-0-147-141:1]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=1]: [After model building] Memory usage: 2619.49MB. Peak reserved memory: 3064.00MB [ip-26-0-147-141:0]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=0]: Number of parameters: 1373642752 (2620.02MB). Expecting peak 4*param_size=10480.06MB with grads and Adam optim states (w/o memory optims) [ip-26-0-147-141:0]:06/15/2023 12:30:47 [INFO|DP=0|PP=0|TP=0]: [After model building] Memory usage: 2620.03MB. Peak reserved memory: 3066.00MB [ip-26-0-147-141:2]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=2]: Global rank: 2/128 | PP: 0/1 | DP: 0/32 | TP: 2/4 [ip-26-0-147-141:1]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=1]: Global rank: 1/128 | PP: 0/1 | DP: 0/32 | TP: 1/4 [ip-26-0-147-141:3]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=3]: Global rank: 3/128 | PP: 0/1 | DP: 0/32 | TP: 3/4 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: Global rank: 0/128 | PP: 0/1 | DP: 0/32 | TP: 0/4 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: Using Nemo Dataloader [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: Building GPT datasets. [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:4]:06/15/2023 12:30:51 [INFO|DP=1|PP=0|TP=0]: Global rank: 4/128 | PP: 0/1 | DP: 1/32 | TP: 0/4 [ip-26-0-147-141:5]:06/15/2023 12:30:51 [INFO|DP=1|PP=0|TP=1]: Global rank: 5/128 | PP: 0/1 | DP: 1/32 | TP: 1/4 [ip-26-0-147-233:0]:06/15/2023 12:30:51 [INFO|DP=10|PP=0|TP=0]: Global rank: 40/128 | PP: 0/1 | DP: 10/32 | TP: 0/4 [ip-26-0-147-233:4]:06/15/2023 12:30:51 [INFO|DP=11|PP=0|TP=0]: Global rank: 44/128 | PP: 0/1 | DP: 11/32 | TP: 0/4 [ip-26-0-147-233:5]:06/15/2023 12:30:51 [INFO|DP=11|PP=0|TP=1]: Global rank: 45/128 | PP: 0/1 | DP: 11/32 | TP: 1/4 [ip-26-0-148-55:2]:06/15/2023 12:30:51 [INFO|DP=26|PP=0|TP=2]: Global rank: 106/128 | PP: 0/1 | DP: 26/32 | TP: 2/4 [ip-26-0-147-204:6]:06/15/2023 12:30:51 [INFO|DP=9|PP=0|TP=2]: Global rank: 38/128 | PP: 0/1 | DP: 9/32 | TP: 2/4 [ip-26-0-148-170:3]:06/15/2023 12:30:51 [INFO|DP=20|PP=0|TP=3]: Global rank: 83/128 | PP: 0/1 | DP: 20/32 | TP: 3/4 [ip-26-0-148-170:4]:06/15/2023 12:30:51 [INFO|DP=21|PP=0|TP=0]: Global rank: 84/128 | PP: 0/1 | DP: 21/32 | TP: 0/4 [ip-26-0-148-245:1]:06/15/2023 12:30:51 [INFO|DP=24|PP=0|TP=1]: Global rank: 97/128 | PP: 0/1 | DP: 24/32 | TP: 1/4 [ip-26-0-147-233:3]:06/15/2023 12:30:51 [INFO|DP=10|PP=0|TP=3]: Global rank: 43/128 | PP: 0/1 | DP: 10/32 | TP: 3/4 [ip-26-0-148-245:0]:06/15/2023 12:30:51 [INFO|DP=24|PP=0|TP=0]: Global rank: 96/128 | PP: 0/1 | DP: 24/32 | TP: 0/4 [ip-26-0-148-245:4]:06/15/2023 12:30:51 [INFO|DP=25|PP=0|TP=0]: Global rank: 100/128 | PP: 0/1 | DP: 25/32 | TP: 0/4 [ip-26-0-148-55:3]:06/15/2023 12:30:51 [INFO|DP=26|PP=0|TP=3]: Global rank: 107/128 | PP: 0/1 | DP: 26/32 | TP: 3/4 [ip-26-0-148-245:3]:06/15/2023 12:30:51 [INFO|DP=24|PP=0|TP=3]: Global rank: 99/128 | PP: 0/1 | DP: 24/32 | TP: 3/4 [ip-26-0-148-245:2]:06/15/2023 12:30:51 [INFO|DP=24|PP=0|TP=2]: Global rank: 98/128 | PP: 0/1 | DP: 24/32 | TP: 2/4 [ip-26-0-148-245:7]:06/15/2023 12:30:51 [INFO|DP=25|PP=0|TP=3]: Global rank: 103/128 | PP: 0/1 | DP: 25/32 | TP: 3/4 [ip-26-0-147-233:2]:06/15/2023 12:30:51 [INFO|DP=10|PP=0|TP=2]: Global rank: 42/128 | PP: 0/1 | DP: 10/32 | TP: 2/4 [ip-26-0-148-245:5]:06/15/2023 12:30:51 [INFO|DP=25|PP=0|TP=1]: Global rank: 101/128 | PP: 0/1 | DP: 25/32 | TP: 1/4 [ip-26-0-147-204:0]:06/15/2023 12:30:51 [INFO|DP=8|PP=0|TP=0]: Global rank: 32/128 | PP: 0/1 | DP: 8/32 | TP: 0/4 [ip-26-0-147-204:1]:06/15/2023 12:30:51 [INFO|DP=8|PP=0|TP=1]: Global rank: 33/128 | PP: 0/1 | DP: 8/32 | TP: 1/4 [ip-26-0-149-1:7]:06/15/2023 12:30:51 [INFO|DP=31|PP=0|TP=3]: Global rank: 127/128 | PP: 0/1 | DP: 31/32 | TP: 3/4 [ip-26-0-147-233:7]:06/15/2023 12:30:51 [INFO|DP=11|PP=0|TP=3]: Global rank: 47/128 | PP: 0/1 | DP: 11/32 | TP: 3/4 [ip-26-0-147-204:2]:06/15/2023 12:30:51 [INFO|DP=8|PP=0|TP=2]: Global rank: 34/128 | PP: 0/1 | DP: 8/32 | TP: 2/4 [ip-26-0-147-204:7]:06/15/2023 12:30:51 [INFO|DP=9|PP=0|TP=3]: Global rank: 39/128 | PP: 0/1 | DP: 9/32 | TP: 3/4 [ip-26-0-147-233:6]:06/15/2023 12:30:51 [INFO|DP=11|PP=0|TP=2]: Global rank: 46/128 | PP: 0/1 | DP: 11/32 | TP: 2/4 [ip-26-0-147-204:3]:06/15/2023 12:30:51 [INFO|DP=8|PP=0|TP=3]: Global rank: 35/128 | PP: 0/1 | DP: 8/32 | TP: 3/4 [ip-26-0-149-1:6]:06/15/2023 12:30:51 [INFO|DP=31|PP=0|TP=2]: Global rank: 126/128 | PP: 0/1 | DP: 31/32 | TP: 2/4 [ip-26-0-148-55:1]:06/15/2023 12:30:51 [INFO|DP=26|PP=0|TP=1]: Global rank: 105/128 | PP: 0/1 | DP: 26/32 | TP: 1/4 [ip-26-0-148-55:0]:06/15/2023 12:30:51 [INFO|DP=26|PP=0|TP=0]: Global rank: 104/128 | PP: 0/1 | DP: 26/32 | TP: 0/4 [ip-26-0-149-1:3]:06/15/2023 12:30:51 [INFO|DP=30|PP=0|TP=3]: Global rank: 123/128 | PP: 0/1 | DP: 30/32 | TP: 3/4 [ip-26-0-149-1:1]:06/15/2023 12:30:51 [INFO|DP=30|PP=0|TP=1]: Global rank: 121/128 | PP: 0/1 | DP: 30/32 | TP: 1/4 [ip-26-0-148-170:6]:06/15/2023 12:30:51 [INFO|DP=21|PP=0|TP=2]: Global rank: 86/128 | PP: 0/1 | DP: 21/32 | TP: 2/4 [ip-26-0-149-1:5]:06/15/2023 12:30:51 [INFO|DP=31|PP=0|TP=1]: Global rank: 125/128 | PP: 0/1 | DP: 31/32 | TP: 1/4 [ip-26-0-148-170:2]:06/15/2023 12:30:51 [INFO|DP=20|PP=0|TP=2]: Global rank: 82/128 | PP: 0/1 | DP: 20/32 | TP: 2/4 [ip-26-0-148-170:1]:06/15/2023 12:30:51 [INFO|DP=20|PP=0|TP=1]: Global rank: 81/128 | PP: 0/1 | DP: 20/32 | TP: 1/4 [ip-26-0-148-170:7]:06/15/2023 12:30:51 [INFO|DP=21|PP=0|TP=3]: Global rank: 87/128 | PP: 0/1 | DP: 21/32 | TP: 3/4 [ip-26-0-148-170:5]:06/15/2023 12:30:51 [INFO|DP=21|PP=0|TP=1]: Global rank: 85/128 | PP: 0/1 | DP: 21/32 | TP: 1/4 [ip-26-0-149-1:2]:06/15/2023 12:30:51 [INFO|DP=30|PP=0|TP=2]: Global rank: 122/128 | PP: 0/1 | DP: 30/32 | TP: 2/4 [ip-26-0-148-170:0]:06/15/2023 12:30:51 [INFO|DP=20|PP=0|TP=0]: Global rank: 80/128 | PP: 0/1 | DP: 20/32 | TP: 0/4 [ip-26-0-149-1:0]:06/15/2023 12:30:51 [INFO|DP=30|PP=0|TP=0]: Global rank: 120/128 | PP: 0/1 | DP: 30/32 | TP: 0/4 [ip-26-0-149-1:4]:06/15/2023 12:30:51 [INFO|DP=31|PP=0|TP=0]: Global rank: 124/128 | PP: 0/1 | DP: 31/32 | TP: 0/4 [ip-26-0-147-233:1]:06/15/2023 12:30:51 [INFO|DP=10|PP=0|TP=1]: Global rank: 41/128 | PP: 0/1 | DP: 10/32 | TP: 1/4 [ip-26-0-148-55:4]:06/15/2023 12:30:51 [INFO|DP=27|PP=0|TP=0]: Global rank: 108/128 | PP: 0/1 | DP: 27/32 | TP: 0/4 [ip-26-0-148-55:7]:06/15/2023 12:30:51 [INFO|DP=27|PP=0|TP=3]: Global rank: 111/128 | PP: 0/1 | DP: 27/32 | TP: 3/4 [ip-26-0-148-55:6]:06/15/2023 12:30:51 [INFO|DP=27|PP=0|TP=2]: Global rank: 110/128 | PP: 0/1 | DP: 27/32 | TP: 2/4 [ip-26-0-147-187:3]:06/15/2023 12:30:51 [INFO|DP=2|PP=0|TP=3]: Global rank: 11/128 | PP: 0/1 | DP: 2/32 | TP: 3/4 [ip-26-0-147-187:5]:06/15/2023 12:30:51 [INFO|DP=3|PP=0|TP=1]: Global rank: 13/128 | PP: 0/1 | DP: 3/32 | TP: 1/4 [ip-26-0-148-55:5]:06/15/2023 12:30:51 [INFO|DP=27|PP=0|TP=1]: Global rank: 109/128 | PP: 0/1 | DP: 27/32 | TP: 1/4 [ip-26-0-147-204:4]:06/15/2023 12:30:51 [INFO|DP=9|PP=0|TP=0]: Global rank: 36/128 | PP: 0/1 | DP: 9/32 | TP: 0/4 [ip-26-0-148-93:7]:06/15/2023 12:30:51 [INFO|DP=29|PP=0|TP=3]: Global rank: 119/128 | PP: 0/1 | DP: 29/32 | TP: 3/4 [ip-26-0-147-187:7]:06/15/2023 12:30:51 [INFO|DP=3|PP=0|TP=3]: Global rank: 15/128 | PP: 0/1 | DP: 3/32 | TP: 3/4 [ip-26-0-147-187:6]:06/15/2023 12:30:51 [INFO|DP=3|PP=0|TP=2]: Global rank: 14/128 | PP: 0/1 | DP: 3/32 | TP: 2/4 [ip-26-0-147-187:0]:06/15/2023 12:30:51 [INFO|DP=2|PP=0|TP=0]: Global rank: 8/128 | PP: 0/1 | DP: 2/32 | TP: 0/4 [ip-26-0-147-187:4]:06/15/2023 12:30:51 [INFO|DP=3|PP=0|TP=0]: Global rank: 12/128 | PP: 0/1 | DP: 3/32 | TP: 0/4 [ip-26-0-147-247:0]:06/15/2023 12:30:51 [INFO|DP=14|PP=0|TP=0]: Global rank: 56/128 | PP: 0/1 | DP: 14/32 | TP: 0/4 [ip-26-0-147-187:1]:06/15/2023 12:30:51 [INFO|DP=2|PP=0|TP=1]: Global rank: 9/128 | PP: 0/1 | DP: 2/32 | TP: 1/4 [ip-26-0-147-187:2]:06/15/2023 12:30:51 [INFO|DP=2|PP=0|TP=2]: Global rank: 10/128 | PP: 0/1 | DP: 2/32 | TP: 2/4 [ip-26-0-148-193:0]:06/15/2023 12:30:51 [INFO|DP=22|PP=0|TP=0]: Global rank: 88/128 | PP: 0/1 | DP: 22/32 | TP: 0/4 [ip-26-0-148-193:3]:06/15/2023 12:30:51 [INFO|DP=22|PP=0|TP=3]: Global rank: 91/128 | PP: 0/1 | DP: 22/32 | TP: 3/4 [ip-26-0-147-204:5]:06/15/2023 12:30:51 [INFO|DP=9|PP=0|TP=1]: Global rank: 37/128 | PP: 0/1 | DP: 9/32 | TP: 1/4 [ip-26-0-148-193:1]:06/15/2023 12:30:51 [INFO|DP=22|PP=0|TP=1]: Global rank: 89/128 | PP: 0/1 | DP: 22/32 | TP: 1/4 [ip-26-0-148-193:2]:06/15/2023 12:30:51 [INFO|DP=22|PP=0|TP=2]: Global rank: 90/128 | PP: 0/1 | DP: 22/32 | TP: 2/4 [ip-26-0-148-193:5]:06/15/2023 12:30:51 [INFO|DP=23|PP=0|TP=1]: Global rank: 93/128 | PP: 0/1 | DP: 23/32 | TP: 1/4 [ip-26-0-148-193:7]:06/15/2023 12:30:51 [INFO|DP=23|PP=0|TP=3]: Global rank: 95/128 | PP: 0/1 | DP: 23/32 | TP: 3/4 [ip-26-0-148-193:6]:06/15/2023 12:30:51 [INFO|DP=23|PP=0|TP=2]: Global rank: 94/128 | PP: 0/1 | DP: 23/32 | TP: 2/4 [ip-26-0-147-247:4]:06/15/2023 12:30:51 [INFO|DP=15|PP=0|TP=0]: Global rank: 60/128 | PP: 0/1 | DP: 15/32 | TP: 0/4 [ip-26-0-147-245:6]:06/15/2023 12:30:51 [INFO|DP=13|PP=0|TP=2]: Global rank: 54/128 | PP: 0/1 | DP: 13/32 | TP: 2/4 [ip-26-0-147-247:1]:06/15/2023 12:30:51 [INFO|DP=14|PP=0|TP=1]: Global rank: 57/128 | PP: 0/1 | DP: 14/32 | TP: 1/4 [ip-26-0-148-151:4]:06/15/2023 12:30:51 [INFO|DP=19|PP=0|TP=0]: Global rank: 76/128 | PP: 0/1 | DP: 19/32 | TP: 0/4 [ip-26-0-147-247:6]:06/15/2023 12:30:51 [INFO|DP=15|PP=0|TP=2]: Global rank: 62/128 | PP: 0/1 | DP: 15/32 | TP: 2/4 [ip-26-0-148-93:5]:06/15/2023 12:30:51 [INFO|DP=29|PP=0|TP=1]: Global rank: 117/128 | PP: 0/1 | DP: 29/32 | TP: 1/4 [ip-26-0-148-193:4]:06/15/2023 12:30:51 [INFO|DP=23|PP=0|TP=0]: Global rank: 92/128 | PP: 0/1 | DP: 23/32 | TP: 0/4 [ip-26-0-148-115:1]:06/15/2023 12:30:51 [INFO|DP=16|PP=0|TP=1]: Global rank: 65/128 | PP: 0/1 | DP: 16/32 | TP: 1/4 [ip-26-0-148-151:2]:06/15/2023 12:30:51 [INFO|DP=18|PP=0|TP=2]: Global rank: 74/128 | PP: 0/1 | DP: 18/32 | TP: 2/4 [ip-26-0-148-245:6]:06/15/2023 12:30:51 [INFO|DP=25|PP=0|TP=2]: Global rank: 102/128 | PP: 0/1 | DP: 25/32 | TP: 2/4 [ip-26-0-148-151:5]:06/15/2023 12:30:51 [INFO|DP=19|PP=0|TP=1]: Global rank: 77/128 | PP: 0/1 | DP: 19/32 | TP: 1/4 [ip-26-0-148-151:6]:06/15/2023 12:30:51 [INFO|DP=19|PP=0|TP=2]: Global rank: 78/128 | PP: 0/1 | DP: 19/32 | TP: 2/4 [ip-26-0-148-151:0]:06/15/2023 12:30:51 [INFO|DP=18|PP=0|TP=0]: Global rank: 72/128 | PP: 0/1 | DP: 18/32 | TP: 0/4 [ip-26-0-147-245:4]:06/15/2023 12:30:51 [INFO|DP=13|PP=0|TP=0]: Global rank: 52/128 | PP: 0/1 | DP: 13/32 | TP: 0/4 [ip-26-0-147-245:0]:06/15/2023 12:30:51 [INFO|DP=12|PP=0|TP=0]: Global rank: 48/128 | PP: 0/1 | DP: 12/32 | TP: 0/4 [ip-26-0-147-247:3]:06/15/2023 12:30:51 [INFO|DP=14|PP=0|TP=3]: Global rank: 59/128 | PP: 0/1 | DP: 14/32 | TP: 3/4 [ip-26-0-148-93:0]:06/15/2023 12:30:51 [INFO|DP=28|PP=0|TP=0]: Global rank: 112/128 | PP: 0/1 | DP: 28/32 | TP: 0/4 [ip-26-0-147-245:1]:06/15/2023 12:30:51 [INFO|DP=12|PP=0|TP=1]: Global rank: 49/128 | PP: 0/1 | DP: 12/32 | TP: 1/4 [ip-26-0-147-247:5]:06/15/2023 12:30:51 [INFO|DP=15|PP=0|TP=1]: Global rank: 61/128 | PP: 0/1 | DP: 15/32 | TP: 1/4 [ip-26-0-148-93:6]:06/15/2023 12:30:51 [INFO|DP=29|PP=0|TP=2]: Global rank: 118/128 | PP: 0/1 | DP: 29/32 | TP: 2/4 [ip-26-0-148-93:2]:06/15/2023 12:30:51 [INFO|DP=28|PP=0|TP=2]: Global rank: 114/128 | PP: 0/1 | DP: 28/32 | TP: 2/4 [ip-26-0-147-247:2]:06/15/2023 12:30:51 [INFO|DP=14|PP=0|TP=2]: Global rank: 58/128 | PP: 0/1 | DP: 14/32 | TP: 2/4 [ip-26-0-148-93:3]:06/15/2023 12:30:51 [INFO|DP=28|PP=0|TP=3]: Global rank: 115/128 | PP: 0/1 | DP: 28/32 | TP: 3/4 [ip-26-0-148-151:1]:06/15/2023 12:30:51 [INFO|DP=18|PP=0|TP=1]: Global rank: 73/128 | PP: 0/1 | DP: 18/32 | TP: 1/4 [ip-26-0-147-247:7]:06/15/2023 12:30:51 [INFO|DP=15|PP=0|TP=3]: Global rank: 63/128 | PP: 0/1 | DP: 15/32 | TP: 3/4 [ip-26-0-148-151:3]:06/15/2023 12:30:51 [INFO|DP=18|PP=0|TP=3]: Global rank: 75/128 | PP: 0/1 | DP: 18/32 | TP: 3/4 [ip-26-0-148-151:7]:06/15/2023 12:30:51 [INFO|DP=19|PP=0|TP=3]: Global rank: 79/128 | PP: 0/1 | DP: 19/32 | TP: 3/4 [ip-26-0-148-93:1]:06/15/2023 12:30:51 [INFO|DP=28|PP=0|TP=1]: Global rank: 113/128 | PP: 0/1 | DP: 28/32 | TP: 1/4 [ip-26-0-147-193:2]:06/15/2023 12:30:51 [INFO|DP=6|PP=0|TP=2]: Global rank: 26/128 | PP: 0/1 | DP: 6/32 | TP: 2/4 [ip-26-0-147-193:0]:06/15/2023 12:30:51 [INFO|DP=6|PP=0|TP=0]: Global rank: 24/128 | PP: 0/1 | DP: 6/32 | TP: 0/4 [ip-26-0-147-193:1]:06/15/2023 12:30:51 [INFO|DP=6|PP=0|TP=1]: Global rank: 25/128 | PP: 0/1 | DP: 6/32 | TP: 1/4 [ip-26-0-148-93:4]:06/15/2023 12:30:51 [INFO|DP=29|PP=0|TP=0]: Global rank: 116/128 | PP: 0/1 | DP: 29/32 | TP: 0/4 [ip-26-0-147-193:5]:06/15/2023 12:30:51 [INFO|DP=7|PP=0|TP=1]: Global rank: 29/128 | PP: 0/1 | DP: 7/32 | TP: 1/4 [ip-26-0-147-193:6]:06/15/2023 12:30:51 [INFO|DP=7|PP=0|TP=2]: Global rank: 30/128 | PP: 0/1 | DP: 7/32 | TP: 2/4 [ip-26-0-147-193:4]:06/15/2023 12:30:51 [INFO|DP=7|PP=0|TP=0]: Global rank: 28/128 | PP: 0/1 | DP: 7/32 | TP: 0/4 [ip-26-0-147-193:7]:06/15/2023 12:30:51 [INFO|DP=7|PP=0|TP=3]: Global rank: 31/128 | PP: 0/1 | DP: 7/32 | TP: 3/4 [ip-26-0-147-193:3]:06/15/2023 12:30:51 [INFO|DP=6|PP=0|TP=3]: Global rank: 27/128 | PP: 0/1 | DP: 6/32 | TP: 3/4 [ip-26-0-147-245:3]:06/15/2023 12:30:51 [INFO|DP=12|PP=0|TP=3]: Global rank: 51/128 | PP: 0/1 | DP: 12/32 | TP: 3/4 [ip-26-0-147-245:2]:06/15/2023 12:30:51 [INFO|DP=12|PP=0|TP=2]: Global rank: 50/128 | PP: 0/1 | DP: 12/32 | TP: 2/4 [ip-26-0-148-115:5]:06/15/2023 12:30:51 [INFO|DP=17|PP=0|TP=1]: Global rank: 69/128 | PP: 0/1 | DP: 17/32 | TP: 1/4 [ip-26-0-147-245:5]:06/15/2023 12:30:51 [INFO|DP=13|PP=0|TP=1]: Global rank: 53/128 | PP: 0/1 | DP: 13/32 | TP: 1/4 [ip-26-0-148-115:6]:06/15/2023 12:30:51 [INFO|DP=17|PP=0|TP=2]: Global rank: 70/128 | PP: 0/1 | DP: 17/32 | TP: 2/4 [ip-26-0-148-115:7]:06/15/2023 12:30:51 [INFO|DP=17|PP=0|TP=3]: Global rank: 71/128 | PP: 0/1 | DP: 17/32 | TP: 3/4 [ip-26-0-147-245:7]:06/15/2023 12:30:51 [INFO|DP=13|PP=0|TP=3]: Global rank: 55/128 | PP: 0/1 | DP: 13/32 | TP: 3/4 [ip-26-0-148-115:2]:06/15/2023 12:30:51 [INFO|DP=16|PP=0|TP=2]: Global rank: 66/128 | PP: 0/1 | DP: 16/32 | TP: 2/4 [ip-26-0-148-115:0]:06/15/2023 12:30:51 [INFO|DP=16|PP=0|TP=0]: Global rank: 64/128 | PP: 0/1 | DP: 16/32 | TP: 0/4 [ip-26-0-148-115:3]:06/15/2023 12:30:51 [INFO|DP=16|PP=0|TP=3]: Global rank: 67/128 | PP: 0/1 | DP: 16/32 | TP: 3/4 [ip-26-0-148-115:4]:06/15/2023 12:30:51 [INFO|DP=17|PP=0|TP=0]: Global rank: 68/128 | PP: 0/1 | DP: 17/32 | TP: 0/4 [ip-26-0-147-189:5]:06/15/2023 12:30:51 [INFO|DP=5|PP=0|TP=1]: Global rank: 21/128 | PP: 0/1 | DP: 5/32 | TP: 1/4 [ip-26-0-147-189:0]:06/15/2023 12:30:51 [INFO|DP=4|PP=0|TP=0]: Global rank: 16/128 | PP: 0/1 | DP: 4/32 | TP: 0/4 [ip-26-0-147-189:3]:06/15/2023 12:30:51 [INFO|DP=4|PP=0|TP=3]: Global rank: 19/128 | PP: 0/1 | DP: 4/32 | TP: 3/4 [ip-26-0-147-189:2]:06/15/2023 12:30:51 [INFO|DP=4|PP=0|TP=2]: Global rank: 18/128 | PP: 0/1 | DP: 4/32 | TP: 2/4 [ip-26-0-147-189:6]:06/15/2023 12:30:51 [INFO|DP=5|PP=0|TP=2]: Global rank: 22/128 | PP: 0/1 | DP: 5/32 | TP: 2/4 [ip-26-0-147-189:1]:06/15/2023 12:30:51 [INFO|DP=4|PP=0|TP=1]: Global rank: 17/128 | PP: 0/1 | DP: 4/32 | TP: 1/4 [ip-26-0-147-189:4]:06/15/2023 12:30:51 [INFO|DP=5|PP=0|TP=0]: Global rank: 20/128 | PP: 0/1 | DP: 5/32 | TP: 0/4 [ip-26-0-147-189:7]:06/15/2023 12:30:51 [INFO|DP=5|PP=0|TP=3]: Global rank: 23/128 | PP: 0/1 | DP: 5/32 | TP: 3/4 [ip-26-0-147-141:6]:06/15/2023 12:30:51 [INFO|DP=1|PP=0|TP=2]: Global rank: 6/128 | PP: 0/1 | DP: 1/32 | TP: 2/4 [ip-26-0-147-141:7]:06/15/2023 12:30:51 [INFO|DP=1|PP=0|TP=3]: Global rank: 7/128 | PP: 0/1 | DP: 1/32 | TP: 3/4 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007547 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 2721616 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 888559) total of 888559 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [888559, 1804629) total of 916070 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [1804629, 2721616) total of 916987 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_train_indexmap_59682ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 192826 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_valid_indexmap_49ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197784 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/css/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 197931 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005378 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 968 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 316) total of 316 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [316, 642) total of 326 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [642, 968) total of 326 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 336 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 135 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/prolog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 157 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004495 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 8536791 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2787113) total of 2787113 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [2787113, 5660514) total of 2873401 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [5660514, 8536791) total of 2876277 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_train_indexmap_1072087ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1619255 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_valid_indexmap_868ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 833054 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 843540 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005430 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 158792 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51843) total of 51843 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [51843, 105291) total of 53448 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [105291, 158792) total of 53501 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_train_indexmap_35412ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53749 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_valid_indexmap_29ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27838 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/fortran/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27546 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001845 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 153194 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 50015) total of 50015 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [50015, 101579) total of 51564 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [101579, 153194) total of 51615 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_train_indexmap_16910ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 22706 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_valid_indexmap_14ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11373 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/solidity/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12126 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001363 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 2239354 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 731110) total of 731110 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [731110, 1484855) total of 753745 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [1484855, 2239354) total of 754499 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_train_indexmap_112998ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 115558 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_valid_indexmap_92ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59022 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/kotlin/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 59162 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003725 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 523 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 171) total of 171 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [171, 347) total of 176 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [347, 523) total of 176 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 255 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 75 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001376 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: number of documents: 295364 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 96431) total of 96431 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [96431, 195848) total of 99417 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: document indices in [195848, 295364) total of 99516 documents [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38582 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:51 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19483 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/julia/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 19724 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003905 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 210816 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 68827) total of 68827 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [68827, 139786) total of 70959 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [139786, 210816) total of 71030 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_train_indexmap_19497ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 23179 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11792 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java-server-pages/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11881 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000520 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 5001 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1633) total of 1633 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [1633, 3316) total of 1683 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [3316, 5001) total of 1685 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_train_indexmap_1592ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2565 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1255 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/isabelle/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1256 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002975 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 8042 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2625) total of 2625 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [2625, 5332) total of 2707 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [5332, 8042) total of 2710 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 797 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 397 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/idris/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 395 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002245 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 16870 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5508) total of 5508 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [5508, 11186) total of 5678 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [11186, 16870) total of 5684 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_train_indexmap_1791ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3142 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.047 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1568 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lean/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1522 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003532 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 267627 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 87375) total of 87375 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [87375, 177456) total of 90081 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [177456, 267627) total of 90171 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_train_indexmap_22282ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 22581 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_valid_indexmap_19ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12030 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/powershell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11686 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002976 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 4700526 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1534640) total of 1534640 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [1534640, 3116791) total of 1582151 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [3116791, 4700526) total of 1583735 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_train_indexmap_473079ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 679828 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_valid_indexmap_383ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 350499 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/go/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 349999 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001572 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 98447 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32142) total of 32142 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [32142, 65278) total of 33136 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [65278, 98447) total of 33169 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_train_indexmap_13926ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 17794 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9455 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/erlang/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9199 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003671 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 124066 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40506) total of 40506 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [40506, 82265) total of 41759 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [82265, 124066) total of 41801 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_train_indexmap_12136ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12957 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6943 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/f-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6521 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002775 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 30934 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10099) total of 10099 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [10099, 20511) total of 10412 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [20511, 30934) total of 10423 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_train_indexmap_5173ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5597 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2939 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ada/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2818 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004059 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 110981 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 36233) total of 36233 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [36233, 73588) total of 37355 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [73588, 110981) total of 37393 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_train_indexmap_33422ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53606 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_valid_indexmap_28ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29260 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/pascal/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27777 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003027 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 365491 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 119326) total of 119326 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [119326, 242347) total of 123021 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [242347, 365491) total of 123144 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 64332 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33791 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/perl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 33144 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001456 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 39042 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 12747) total of 12747 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [12747, 25888) total of 13141 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [25888, 39042) total of 13154 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8357 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4310 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/r/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4206 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003807 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 97167 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31724) total of 31724 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [31724, 64429) total of 32705 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [64429, 97167) total of 32738 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_train_indexmap_6168ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7976 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4021 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/protocol-buffer/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4218 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002849 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 186375 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 60848) total of 60848 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [60848, 123580) total of 62732 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [123580, 186375) total of 62795 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_train_indexmap_8953ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 12190 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6104 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cmake/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6249 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003724 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 9226 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3013) total of 3013 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [3013, 6118) total of 3105 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [6118, 9226) total of 3108 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_train_indexmap_2388ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4469 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1942 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2227 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002981 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: number of documents: 3390320 [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1106880) total of 1106880 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [1106880, 2248029) total of 1141149 documents [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:52 [INFO|DP=0|PP=0|TP=0]: document indices in [2248029, 3390320) total of 1142291 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_train_indexmap_135479ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 161899 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_valid_indexmap_110ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81430 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ruby/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 82121 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003843 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 1380468 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 450699) total of 450699 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [450699, 915351) total of 464652 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [915351, 1380468) total of 465117 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_train_indexmap_181235ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 214187 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_valid_indexmap_147ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110422 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rust/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 110680 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005570 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 5386 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1758) total of 1758 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [1758, 3571) total of 1813 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [3571, 5386) total of 1815 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_train_indexmap_1194ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1584 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 816 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/rmarkdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002609 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 10801285 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3526430) total of 3526430 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [3526430, 7162038) total of 3635608 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [7162038, 10801285) total of 3639247 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_train_indexmap_888466ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1254025 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_valid_indexmap_720ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 430672 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/c-sharp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 429969 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003148 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 587748 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 191890) total of 191890 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [191890, 389720) total of 197830 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [389720, 587748) total of 198028 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_train_indexmap_11539ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15786 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7945 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/smalltalk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8334 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005347 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 541454 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 176775) total of 176775 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [176775, 359023) total of 182248 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [359023, 541454) total of 182431 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_train_indexmap_44364ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 52419 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_valid_indexmap_36ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26827 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 26582 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001172 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 1152 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 376) total of 376 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [376, 764) total of 388 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [764, 1152) total of 388 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 209 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.003 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 71 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/maple/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 58 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004166 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 22653 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7396) total of 7396 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [7396, 15021) total of 7625 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [15021, 22653) total of 7632 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_train_indexmap_24868ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 39578 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_valid_indexmap_21ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21128 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/mathematica/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21473 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005090 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 158356 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 51701) total of 51701 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [51701, 105002) total of 53301 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [105002, 158356) total of 53354 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_train_indexmap_20491ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 27268 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13487 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/ocaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13675 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003178 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 657349 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 214613) total of 214613 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [214613, 435870) total of 221257 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [435870, 657349) total of 221479 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_train_indexmap_26062ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 39676 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_valid_indexmap_22ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20452 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/makefile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20653 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001327 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 549459 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 179388) total of 179388 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [179388, 364331) total of 184943 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [364331, 549459) total of 185128 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_train_indexmap_57096ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81936 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_valid_indexmap_47ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 41600 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/lua/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42207 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002342 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 1133 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 370) total of 370 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [370, 751) total of 381 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [751, 1133) total of 382 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 207 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001747 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 6104 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1992) total of 1992 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [1992, 4047) total of 2055 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [4047, 6104) total of 2057 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1341 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 696 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/literate-haskell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 660 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003541 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 896880 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 292816) total of 292816 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [292816, 594697) total of 301881 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [594697, 896880) total of 302183 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_train_indexmap_66049ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 84311 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_valid_indexmap_54ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42767 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/restructuredtext/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 42618 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003689 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 3688 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1204) total of 1204 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [1204, 2445) total of 1241 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [2445, 3688) total of 1243 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 891 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 3 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 347 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/racket/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 301 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005264 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 19630 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6409) total of 6409 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [6409, 13016) total of 6607 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [13016, 19630) total of 6614 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_train_indexmap_3780ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5494 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2812 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/standard-ml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2578 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005189 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 46270 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 15106) total of 15106 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [15106, 30680) total of 15574 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [30680, 46270) total of 15590 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_train_indexmap_7759ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11740 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6106 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/systemverilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6324 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003554 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: number of documents: 522778 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 170678) total of 170678 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [170678, 346640) total of 175962 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: document indices in [346640, 522778) total of 176138 documents [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_train_indexmap_103449ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 149597 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_valid_indexmap_84ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of samples: 78658 [ip-26-0-147-141:0]:06/15/2023 12:30:53 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tex/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 77461 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004015 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 10289 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3359) total of 3359 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3359, 6822) total of 3463 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [6822, 10289) total of 3467 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 623 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 377 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/awk/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 312 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002190 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 247919 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 80941) total of 80941 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [80941, 164388) total of 83447 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [164388, 247919) total of 83531 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_train_indexmap_31035ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32911 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_valid_indexmap_26ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 31936 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/assembly/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 32667 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005691 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 5368 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1752) total of 1752 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1752, 3559) total of 1807 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3559, 5368) total of 1809 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 253 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 137 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/alloy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 117 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002181 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 17554 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5731) total of 5731 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [5731, 11640) total of 5909 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [11640, 17554) total of 5914 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_train_indexmap_1393ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2751 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1328 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/agda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1277 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000737 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 52838 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 17250) total of 17250 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [17250, 35035) total of 17785 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [35035, 52838) total of 17803 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_train_indexmap_8157ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10038 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5153 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/emacs-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5279 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003164 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 928415 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 303112) total of 303112 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [303112, 615607) total of 312495 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [615607, 928415) total of 312808 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_train_indexmap_72812ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 74412 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_valid_indexmap_59ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38815 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dart/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 38423 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002524 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 58151 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 18985) total of 18985 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [18985, 38558) total of 19573 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [38558, 58151) total of 19593 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_train_indexmap_11141ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15246 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_valid_indexmap_10ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7892 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cuda/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7791 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003203 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 5928 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1936) total of 1936 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1936, 3931) total of 1995 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3931, 5928) total of 1997 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_train_indexmap_597ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 801 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 370 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/bluespec/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 510 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001833 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 180 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 58) total of 58 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [58, 119) total of 61 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [119, 180) total of 61 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.005 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 21 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/augeas/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002642 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 239568 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 78215) total of 78215 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [78215, 158851) total of 80636 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [158851, 239568) total of 80717 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_train_indexmap_4576ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7142 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4062 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/batchfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4032 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004004 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 4806 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1569) total of 1569 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1569, 3187) total of 1618 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3187, 4806) total of 1619 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_train_indexmap_398ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 524 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 254 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcsh/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 284 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.000595 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 5429 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1773) total of 1773 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1773, 3600) total of 1827 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3600, 5429) total of 1829 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 395 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 238 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stan/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 215 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001752 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 1355788 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 442641) total of 442641 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [442641, 898986) total of 456345 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [898986, 1355788) total of 456802 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_train_indexmap_93303ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103367 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_valid_indexmap_76ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53519 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scala/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 53423 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002774 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 49335 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 16107) total of 16107 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [16107, 32713) total of 16606 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [32713, 49335) total of 16622 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_train_indexmap_6963ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9597 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5123 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/tcl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5055 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003540 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 24208 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 7904) total of 7904 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [7904, 16052) total of 8148 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [16052, 24208) total of 8156 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_train_indexmap_6566ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7506 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_valid_indexmap_6ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9094 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/stata/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9893 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003324 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 4737 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1547) total of 1547 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1547, 3141) total of 1594 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [3141, 4737) total of 1596 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 221 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 103 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/applescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 102 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002900 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 2206327 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 720327) total of 720327 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [720327, 1462955) total of 742628 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [1462955, 2206327) total of 743372 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_train_indexmap_61473ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86412 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_valid_indexmap_50ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44088 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/shell/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 44638 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002965 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 125163 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 40863) total of 40863 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [40863, 82992) total of 42129 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [82992, 125163) total of 42171 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_train_indexmap_9152ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.042 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 10752 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_valid_indexmap_8ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5432 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/clojure/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5607 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003240 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 41890 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13676) total of 13676 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [13676, 27776) total of 14100 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [27776, 41890) total of 14114 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_train_indexmap_3979ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5078 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_valid_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2583 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/scheme/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2873 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001769 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 7917 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2585) total of 2585 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [2585, 5250) total of 2665 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [5250, 7917) total of 2667 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1503 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/antlr/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 720 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003358 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 13716 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4478) total of 4478 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [4478, 9095) total of 4617 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [9095, 13716) total of 4621 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_train_indexmap_796ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1284 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 678 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sparql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of samples: 622 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001230 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: number of documents: 975420 [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 318457) total of 318457 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [318457, 646774) total of 328317 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: document indices in [646774, 975420) total of 328646 documents [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:54 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_train_indexmap_220625ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220816 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_valid_indexmap_179ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 220204 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/sql/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 223151 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005157 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 167701 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 54751) total of 54751 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [54751, 111198) total of 56447 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [111198, 167701) total of 56503 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_train_indexmap_7958ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 13241 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7937 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/glsl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6885 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001966 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 62033 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 20252) total of 20252 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [20252, 41132) total of 20880 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [41132, 62033) total of 20901 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_train_indexmap_5969ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6169 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_valid_indexmap_5ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 3168 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elm/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2971 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002170 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 571506 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 186587) total of 186587 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [186587, 378950) total of 192363 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [378950, 571506) total of 192556 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_train_indexmap_8356ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 11744 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_valid_indexmap_7ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6044 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/dockerfile/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6002 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003326 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 6353527 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2074315) total of 2074315 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [2074315, 4212851) total of 2138536 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [4212851, 6353527) total of 2140676 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_train_indexmap_973214ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1288594 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_valid_indexmap_788ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 662447 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/cpp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 667385 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003089 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 226209 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 73853) total of 73853 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [73853, 149993) total of 76140 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [149993, 226209) total of 76216 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_train_indexmap_12733ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 14695 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_valid_indexmap_11ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7544 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/coffeescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 7711 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003397 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 98733 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 32234) total of 32234 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [32234, 65467) total of 33233 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [65467, 98733) total of 33266 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_train_indexmap_27852ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 37898 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20792 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/common-lisp/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 20468 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004076 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 281016 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 91747) total of 91747 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [91747, 186334) total of 94587 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [186334, 281016) total of 94682 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_train_indexmap_14125ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 17928 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_valid_indexmap_12ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8898 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/elixir/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8929 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003543 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 250834 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 81893) total of 81893 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [81893, 166321) total of 84428 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [166321, 250834) total of 84513 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_train_indexmap_18104ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 18411 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_valid_indexmap_15ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9369 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/groovy/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 9429 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003602 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 3299965 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1077381) total of 1077381 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [1077381, 2188117) total of 1110736 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [2188117, 3299965) total of 1111848 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_train_indexmap_584088ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 783002 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_valid_indexmap_473ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 404400 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/html/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 405099 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002664 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 20071773 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6553082) total of 6553082 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [6553082, 13309046) total of 6755964 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [13309046, 20071773) total of 6762727 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_train_indexmap_1729583ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1806678 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_valid_indexmap_1400ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 930741 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/java/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 933066 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.006462 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 19544285 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6380866) total of 6380866 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [6380866, 12959283) total of 6578417 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [12959283, 19544285) total of 6585002 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_train_indexmap_1287340ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1510953 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_valid_indexmap_1042ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776848 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/javascript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of samples: 776230 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003144 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: number of documents: 21029287 [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 6865694) total of 6865694 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [6865694, 13943948) total of 7078254 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: document indices in [13943948, 21029287) total of 7085339 documents [ip-26-0-147-141:0]:06/15/2023 12:30:55 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_train_indexmap_1490657ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2022322 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_valid_indexmap_1207ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1046848 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/markdown/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1047246 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004650 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 15683017 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5120231) total of 5120231 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [5120231, 10398982) total of 5278751 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [10398982, 15683017) total of 5284035 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_train_indexmap_1211345ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1341900 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_valid_indexmap_981ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 690231 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/php/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 692824 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004578 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 12866649 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 4200736) total of 4200736 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [4200736, 8531525) total of 4330789 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [8531525, 12866649) total of 4335124 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_train_indexmap_1201597ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1407216 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_valid_indexmap_973ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 725128 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/python/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 723779 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.007845 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 10547331 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 3443519) total of 3443519 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [3443519, 6993648) total of 3550129 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [6993648, 10547331) total of 3553683 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_train_indexmap_527589ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 592637 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_valid_indexmap_428ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 303302 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/typescript/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 304761 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004281 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 75 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 25) total of 25 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [25, 50) total of 25 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [50, 75) total of 25 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_train_indexmap_20ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 24 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 4 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 8 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/verilog/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002667 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 161239 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 52642) total of 52642 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [52642, 106913) total of 54271 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [106913, 161239) total of 54326 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_train_indexmap_28250ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29718 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_valid_indexmap_23ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15235 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/visual-basic/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15594 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004060 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 58208 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 19004) total of 19004 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [19004, 38596) total of 19592 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [38596, 58208) total of 19612 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_train_indexmap_18701ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 29022 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_valid_indexmap_16ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 15492 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/vhdl/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 16244 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004706 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 4661 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1522) total of 1522 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [1522, 3091) total of 1569 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [3091, 4661) total of 1570 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_train_indexmap_199ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 300 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.003 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 149 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/thrift/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 138 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001456 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 93 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 31) total of 31 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [31, 62) total of 31 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [62, 93) total of 31 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_train_indexmap_4ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.011 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 6 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/matlab/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.001814 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 7451 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2433) total of 2433 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [2433, 4941) total of 2508 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [4941, 7451) total of 2510 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_train_indexmap_2189ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2982 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_valid_indexmap_2ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1463 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yacc/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1483 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.004159 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: number of documents: 15850 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 5175) total of 5175 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [5175, 10510) total of 5335 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: document indices in [10510, 15850) total of 5340 documents [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_train_indexmap_3581ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of samples: 4266 [ip-26-0-147-141:0]:06/15/2023 12:30:56 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_valid_indexmap_3ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2605 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/zig/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 2298 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003552 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 42103 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 13746) total of 13746 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [13746, 27917) total of 14171 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [27917, 42103) total of 14186 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_train_indexmap_995ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5493 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_valid_indexmap_1ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5972 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/xslt/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 5680 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005045 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 4751547 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1551296) total of 1551296 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [1551296, 3150621) total of 1599325 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [3150621, 4751547) total of 1600926 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.007 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 83479 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86240 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/json/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 86171 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.003220 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 3995948 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 1304607) total of 1304607 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [1304607, 2649604) total of 1344997 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [2649604, 3995948) total of 1346344 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_train_indexmap_19894ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 47956 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_valid_indexmap_17ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49437 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/code/yaml/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 49416 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002154 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 30982955 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 10115392) total of 10115392 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [10115392, 20543954) total of 10428562 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [20543954, 30982955) total of 10439001 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_train_indexmap_1082233ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 1436583 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_valid_indexmap_876ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.012 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 742304 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_issues/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 740971 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002183 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 7634718 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 2492602) total of 2492602 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [2492602, 5062374) total of 2569772 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [5062374, 7634718) total of 2572344 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_train_indexmap_636608ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 647821 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_valid_indexmap_516ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 669748 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/gh_commits/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 665861 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.002710 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 914510 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 298572) total of 298572 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [298572, 606387) total of 307815 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [606387, 914510) total of 308123 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_train_indexmap_141646ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 200647 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_valid_indexmap_115ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.008 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 101464 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_scripts/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 99559 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > building dataset index ... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading sizes... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading pointers... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: reading document index... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating numpy buffer of mmap... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: creating memory view of numpy buffer... [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > finished creating indexed dataset in 0.005184 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: number of documents: 668743 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > dataset split: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: train: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [0, 218333) total of 218333 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: validation: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [218333, 443425) total of 225092 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: test: [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: document indices in [443425, 668743) total of 225318 documents [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_train_indexmap_119364ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.009 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 160224 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 2 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_valid_indexmap_97ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.006 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81239 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading doc-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_doc_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading sample-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_sample_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: > loading shuffle-idx mapping from /fsx/bigcode/bigcode-training/tokenized_stack_no_pii/jupyter_structured/gpt2-preprocessed_content_document_test_indexmap_0ns_8192sl_1234s_shuffle_idx.npy [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: loaded indexed file in 0.010 seconds [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of samples: 81006 [ip-26-0-147-141:0]:06/15/2023 12:30:57 [INFO|DP=0|PP=0|TP=0]: total number of epochs: 1 [ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:1]:make: Nothing to be done for 'default'. [ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:3]:make: Nothing to be done for 'default'. [ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Nothing to be done for 'default'. [ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:make: Nothing to be done for 'default'. [ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:> building indices for blendable datasets ... [ip-26-0-147-141:0]: > sample ratios: [ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: 0.00391158 [ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: 0.0702651 [ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: 0.00232085 [ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: 0.00110829 [ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: 0.00740593 [ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: 0.00170805 [ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: 0.00127778 [ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: 0.00010434 [ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: 3.91276e-05 [ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: 0.000117317 [ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: 0.00146031 [ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: 0.0310058 [ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: 0.000912715 [ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: 0.000795332 [ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: 0.000338975 [ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: 0.00219049 [ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: 0.00290763 [ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: 0.000391145 [ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: 0.000404187 [ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: 0.000586717 [ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: 0.000156445 [ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: 0.00887928 [ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: 0.0118782 [ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: 7.82552e-05 [ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: 0.0582305 [ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: 0.00075627 [ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: 0.00290763 [ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: 0.00162986 [ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: 0.00134299 [ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: 0.00170805 [ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: 0.00374209 [ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: 6.52127e-05 [ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: 0.00432881 [ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: 3.91276e-05 [ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: 0.000247743 [ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: 0.000508528 [ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: 0.00678009 [ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: 2.60851e-05 [ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: 0.00203405 [ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: 9.12977e-05 [ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: 0.000534613 [ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: 0.00477213 [ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: 0.000730185 [ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: 3.91276e-05 [ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: 1.31081e-06 [ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: 0.000299913 [ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: 2.60851e-05 [ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: 0.00611511 [ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: 0.000456358 [ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: 0.000430273 [ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: 0.0040289 [ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: 0.00059976 [ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: 0.000260785 [ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: 6.52127e-05 [ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: 5.21701e-05 [ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: 0.0144598 [ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: 0.00052157 [ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: 0.000391145 [ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: 0.00054759 [ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: 0.0637849 [ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: 0.00083446 [ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: 0.00182543 [ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: 0.000925758 [ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: 0.00118654 [ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: 0.0382814 [ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: 0.113358 [ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: 0.0843729 [ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: 0.0976983 [ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: 0.0793921 [ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: 0.0787532 [ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: 0.0345784 [ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: 1.31081e-06 [ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: 0.00185145 [ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: 0.0012256 [ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: 1.30425e-05 [ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: 2.62161e-07 [ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: 0.000143402 [ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: 0.0002347 [ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: 6.52127e-05 [ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: 0.00130386 [ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: 0.00130386 [ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: 0.0709301 [ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: 0.0417236 [ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: 0.00928347 [ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: 0.00782316 [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 1.73 (sec) [ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:3]:make: Nothing to be done for 'default'. [ip-26-0-147-141:1]:make: Nothing to be done for 'default'. [ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:3]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:1]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Nothing to be done for 'default'. [ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:make: Nothing to be done for 'default'. [ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:> building indices for blendable datasets ... [ip-26-0-147-141:0]: > sample ratios: [ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: 0.00395225 [ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: 0.0702533 [ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: 0.00233909 [ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: 0.00112921 [ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: 0.00742055 [ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: 0.00169382 [ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: 0.00129053 [ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: 0.000161316 [ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: 0.00145185 [ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: 0.0309727 [ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: 0.00088724 [ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: 0.000806582 [ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: 0.000322633 [ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: 0.00217777 [ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: 0.00290369 [ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: 0.000403291 [ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: 0.000403291 [ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: 0.000564607 [ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: 0.000161316 [ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: 0.0088724 [ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: 0.0118568 [ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: 0.0582352 [ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: 0.000725924 [ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: 0.00290369 [ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: 0.00161316 [ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: 0.00137119 [ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: 0.00169382 [ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: 0.00371028 [ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: 0.00435554 [ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: 0.000241975 [ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: 0.000483949 [ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: 0.00677529 [ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: 0.00201645 [ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: 0.000564607 [ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: 0.00475883 [ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: 0.000725924 [ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: 0 [ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: 0.000322633 [ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: 0.00613002 [ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: 0.000483949 [ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: 0.000403291 [ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: 0.00403291 [ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: 0.000645265 [ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: 0.000241975 [ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: 0.0144378 [ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: 0.000564607 [ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: 0.000403291 [ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: 0.000564607 [ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: 0.0638006 [ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: 0.000806582 [ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: 0.00185514 [ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: 0.000967898 [ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: 0.00120987 [ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: 0.0383126 [ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: 0.113325 [ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: 0.0843684 [ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: 0.097677 [ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: 0.0793676 [ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: 0.0787224 [ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: 0.0346024 [ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: 0 [ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: 0.00185514 [ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: 0.00120987 [ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: 0 [ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: 0 [ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: 0.000161316 [ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: 0.000241975 [ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: 8.06582e-05 [ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: 0.00129053 [ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: 0.00129053 [ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: 0.0708985 [ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: 0.0417003 [ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: 0.00927569 [ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: 0.00782384 [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.07 (sec) [ip-26-0-147-141:0]:make: Entering directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:3]:make: Nothing to be done for 'default'. [ip-26-0-147-141:1]:make: Nothing to be done for 'default'. [ip-26-0-147-141:3]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Nothing to be done for 'default'. [ip-26-0-147-141:1]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:2]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:make: Nothing to be done for 'default'. [ip-26-0-147-141:0]:make: Leaving directory '/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/nemo_dataset' [ip-26-0-147-141:0]:> building indices for blendable datasets ... [ip-26-0-147-141:0]: > sample ratios: [ip-26-0-147-141:0]: dataset 0, input: 0.00391159, achieved: -nan [ip-26-0-147-141:0]: dataset 1, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 2, input: 0.0702651, achieved: -nan [ip-26-0-147-141:0]: dataset 3, input: 0.00232087, achieved: -nan [ip-26-0-147-141:0]: dataset 4, input: 0.00110828, achieved: -nan [ip-26-0-147-141:0]: dataset 5, input: 0.00740594, achieved: -nan [ip-26-0-147-141:0]: dataset 6, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 7, input: 0.00170806, achieved: -nan [ip-26-0-147-141:0]: dataset 8, input: 0.00127778, achieved: -nan [ip-26-0-147-141:0]: dataset 9, input: 0.000104309, achieved: -nan [ip-26-0-147-141:0]: dataset 10, input: 3.91159e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 11, input: 0.000117348, achieved: -nan [ip-26-0-147-141:0]: dataset 12, input: 0.00146033, achieved: -nan [ip-26-0-147-141:0]: dataset 13, input: 0.0310058, achieved: -nan [ip-26-0-147-141:0]: dataset 14, input: 0.000912704, achieved: -nan [ip-26-0-147-141:0]: dataset 15, input: 0.000795356, achieved: -nan [ip-26-0-147-141:0]: dataset 16, input: 0.000339004, achieved: -nan [ip-26-0-147-141:0]: dataset 17, input: 0.00219049, achieved: -nan [ip-26-0-147-141:0]: dataset 18, input: 0.00290761, achieved: -nan [ip-26-0-147-141:0]: dataset 19, input: 0.000391159, achieved: -nan [ip-26-0-147-141:0]: dataset 20, input: 0.000404197, achieved: -nan [ip-26-0-147-141:0]: dataset 21, input: 0.000586738, achieved: -nan [ip-26-0-147-141:0]: dataset 22, input: 0.000156463, achieved: -nan [ip-26-0-147-141:0]: dataset 23, input: 0.0088793, achieved: -nan [ip-26-0-147-141:0]: dataset 24, input: 0.0118782, achieved: -nan [ip-26-0-147-141:0]: dataset 25, input: 7.82317e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 26, input: 0.0582305, achieved: -nan [ip-26-0-147-141:0]: dataset 27, input: 0.00075624, achieved: -nan [ip-26-0-147-141:0]: dataset 28, input: 0.00290761, achieved: -nan [ip-26-0-147-141:0]: dataset 29, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 30, input: 0.00162983, achieved: -nan [ip-26-0-147-141:0]: dataset 31, input: 0.00134298, achieved: -nan [ip-26-0-147-141:0]: dataset 32, input: 0.00170806, achieved: -nan [ip-26-0-147-141:0]: dataset 33, input: 0.00374208, achieved: -nan [ip-26-0-147-141:0]: dataset 34, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 35, input: 6.51931e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 36, input: 0.00432882, achieved: -nan [ip-26-0-147-141:0]: dataset 37, input: 3.91159e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 38, input: 0.000247734, achieved: -nan [ip-26-0-147-141:0]: dataset 39, input: 0.000508506, achieved: -nan [ip-26-0-147-141:0]: dataset 40, input: 0.00678008, achieved: -nan [ip-26-0-147-141:0]: dataset 41, input: 2.60772e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 42, input: 0.00203403, achieved: -nan [ip-26-0-147-141:0]: dataset 43, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 44, input: 9.12704e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 45, input: 0.000534584, achieved: -nan [ip-26-0-147-141:0]: dataset 46, input: 0.00477214, achieved: -nan [ip-26-0-147-141:0]: dataset 47, input: 0.000730163, achieved: -nan [ip-26-0-147-141:0]: dataset 48, input: 3.91159e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 49, input: 1.30386e-06, achieved: -nan [ip-26-0-147-141:0]: dataset 50, input: 0.000299888, achieved: -nan [ip-26-0-147-141:0]: dataset 51, input: 2.60772e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 52, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 53, input: 0.00611511, achieved: -nan [ip-26-0-147-141:0]: dataset 54, input: 0.000456352, achieved: -nan [ip-26-0-147-141:0]: dataset 55, input: 0.000430275, achieved: -nan [ip-26-0-147-141:0]: dataset 56, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 57, input: 0.00402893, achieved: -nan [ip-26-0-147-141:0]: dataset 58, input: 0.000599777, achieved: -nan [ip-26-0-147-141:0]: dataset 59, input: 0.000260772, achieved: -nan [ip-26-0-147-141:0]: dataset 60, input: 6.51931e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 61, input: 5.21545e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 62, input: 0.0144598, achieved: -nan [ip-26-0-147-141:0]: dataset 63, input: 0.000521545, achieved: -nan [ip-26-0-147-141:0]: dataset 64, input: 0.000391159, achieved: -nan [ip-26-0-147-141:0]: dataset 65, input: 0.000547622, achieved: -nan [ip-26-0-147-141:0]: dataset 66, input: 0.0637849, achieved: -nan [ip-26-0-147-141:0]: dataset 67, input: 0.000834472, achieved: -nan [ip-26-0-147-141:0]: dataset 68, input: 0.00182541, achieved: -nan [ip-26-0-147-141:0]: dataset 69, input: 0.000925742, achieved: -nan [ip-26-0-147-141:0]: dataset 70, input: 0.00118651, achieved: -nan [ip-26-0-147-141:0]: dataset 71, input: 0.0382814, achieved: -nan [ip-26-0-147-141:0]: dataset 72, input: 0.113358, achieved: -nan [ip-26-0-147-141:0]: dataset 73, input: 0.0843729, achieved: -nan [ip-26-0-147-141:0]: dataset 74, input: 0.0976984, achieved: -nan [ip-26-0-147-141:0]: dataset 75, input: 0.0793922, achieved: -nan [ip-26-0-147-141:0]: dataset 76, input: 0.0787533, achieved: -nan [ip-26-0-147-141:0]: dataset 77, input: 0.0345784, achieved: -nan [ip-26-0-147-141:0]: dataset 78, input: 1.30386e-06, achieved: -nan [ip-26-0-147-141:0]: dataset 79, input: 0.00185148, achieved: -nan [ip-26-0-147-141:0]: dataset 80, input: 0.00122563, achieved: -nan [ip-26-0-147-141:0]: dataset 81, input: 1.30386e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 82, input: 2.60772e-07, achieved: -nan [ip-26-0-147-141:0]: dataset 83, input: 0.000143425, achieved: -nan [ip-26-0-147-141:0]: dataset 84, input: 0.000234695, achieved: -nan [ip-26-0-147-141:0]: dataset 85, input: 6.51931e-05, achieved: -nan [ip-26-0-147-141:0]: dataset 86, input: 0.00130386, achieved: -nan [ip-26-0-147-141:0]: dataset 87, input: 0.00130386, achieved: -nan [ip-26-0-147-141:0]: dataset 88, input: 0.0709301, achieved: -nan [ip-26-0-147-141:0]: dataset 89, input: 0.0417236, achieved: -nan [ip-26-0-147-141:0]: dataset 90, input: 0.0092835, achieved: -nan [ip-26-0-147-141:0]: dataset 91, input: 0.00782317, achieved: -nan [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: > elapsed time for building blendable dataset indices: 0.05 (sec) [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: Building dataloader with consumed samples: 0 [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: Instantiating MegatronPretrainingSampler with total_samples: 15257771 and consumed_samples: 0 [ip-26-0-147-141:0]:06/15/2023 12:30:59 [INFO|DP=0|PP=0|TP=0]: [Before the start of training] datetime: 2023-06-15 12:30:59.604322 [ip-26-0-148-245:1]:Traceback (most recent call last): [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:1]: main() [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:1]: result = model(**micro_batch) [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:1]: sharded_logits = self.model( [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:1]: query_states, kv_states = self.qkv( [ip-26-0-148-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:1]: return column_linear( [ip-26-0-148-245:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:4]:Traceback (most recent call last): [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:4]: main() [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:4]: result = model(**micro_batch) [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:4]: sharded_logits = self.model( [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:4]: query_states, kv_states = self.qkv( [ip-26-0-148-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:4]: return forward_call(*input, **kwargs) [ip-26-0-148-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:4]: return column_linear( [ip-26-0-148-245:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:1]:Traceback (most recent call last): [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:1]: main() [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:1]: result = model(**micro_batch) [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:1]: sharded_logits = self.model( [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:1]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:3]:Traceback (most recent call last): [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:3]: main() [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:3]: result = model(**micro_batch) [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:3]: sharded_logits = self.model( [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:3]: query_states, kv_states = self.qkv( [ip-26-0-148-55:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:3]: return forward_call(*input, **kwargs) [ip-26-0-148-55:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:3]: return column_linear( [ip-26-0-148-55:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-55:1]:Traceback (most recent call last): [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:1]: main() [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:1]: result = model(**micro_batch) [ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:1]: sharded_logits = self.model( [ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/[ip-26-0-149-1:7]:Traceback (most recent call last): [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:7]: main() [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:7]: outputs = pipeline_engine.train_batch_iter( fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:5]:Traceback (most recent call last): [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:5]: main() [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:5]: result = model(**micro_batch) [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/l[ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:1]: query_states, kv_states = self.qkv( [ip-26-0-148-55:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:1]: return column_linear( [ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:7]: result = model(**micro_batch) [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:7]: sharded_logits = self.model( [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:7]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:5]: sharded_logits = self.model( [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-149-1:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:7]: query_states, kv_states = self.qkv( [ip-26-0-149-1:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:7]: return column_linear( [ip-26-0-149-1:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:0]:Traceback (most recent call last): [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:0]: main() [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:0]: result = model(**micro_batch) [ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:0]: sharded_logits = self.model( [ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:6]:Traceback (most recent call last): [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:6]: main() [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:6]: result = model(**micro_batch) [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/exa[ip-26-0-148-170:0]: query_states, kv_states = self.qkv( mples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:6]: sharded_logits = self.model( [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:6]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", li[ip-26-0-148-170:1]:Traceback (most recent call last): ne 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:6]: query_states, kv_states = self.qkv( [ip-26-0-149-1:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:6]: re[ip-26-0-148-170:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:0]: return forward_call(*input, **kwargs) turn column_linear( [ip-26-0-149-1:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:0]: return column_linear( [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:1]: main() [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:1]: result = model(**micro_batch) [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:1]: sharded_logits = self.model( [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]:Traceback (most recent call last): [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:5]: main() [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:5]: result = model(**micro_batch) [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:5]: sharded_logits = self.model( [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-170:5]: query_states, kv_states = self.qkv( [ip-26-0-148-170:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:5]: return forward_call(*input, **kwargs) [ip-26-0-148-170:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:5]: return column_linear( [ip-26-0-148-170:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:0]:Traceback (most recent call last): [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:0]: main() [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:0]: result = model(**micro_batch) [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/l[ip-26-0-148-55:5]:Traceback (most recent call last): [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:5]: main() [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:5]: result = model(**micro_batch) [ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:5]: sharded_logits = self.model( [ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:0]: sharded_logits = self.model( [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:0]: query_states, kv_states = self.qkv( [ip-26-0-148-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:0]: return forward_call(*input, **kwargs) [ip-26-0-148-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-148-55:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ing_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:0]: return column_linear( [ip-26-0-148-193:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:2]:Traceback (most recent call last): [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:2]: main() [ip-26-0-148-93:0]:Traceback (most recent call last): [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:0]: main() [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:0]: result = model(**micro_batch) [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:2]: result = model(**micro_batch) [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:2]: sharded_logits = self.model( [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:2]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:2]: query_states, kv_states = self.qkv( [ip-26-0-148-55:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:2]: return forward_call(*input, **kwargs) [ip-26-0-148-55:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:2]: return column_linear( [ip-26-0-148-55:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:5]:Traceback (most recent call last): [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:5]: main() [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:5]: result = model(**micro_batch) [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:5]: sharded_logits = self.model( [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:5]: query_states, kv_states = self.qkv( [ip-26-0-148-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:5]: return column_linear( [ip-26-0-148-193:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:3]:Traceback (most recent call last): [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:3]: main() [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:3]: result = model(**micro_batch) [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:3]: sharded_logits = self.model( [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:3]: query_states, kv_states = self.qkv( [ip-26-0-148-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:3]: return column_linear( [ip-26-0-148-193:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:4]:Traceback (most recent call last): [ip-26-0-147-247:1]:Traceback (most recent call last): [ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:4]: main() [ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:1]: main() [ip-26-0-147-247:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:5]:Traceback (most recent call last): [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:5]: main() [ip-26-0-147-247:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:1]: result = model(**micro_batch) [ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:5]: result = model(**micro_batch) [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:5]: sharded_logits = self.model( [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:1]: sharded_logits = self.model( [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-[ip-26-0-147-247:4]: result = model(**micro_batch) [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl 26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:5]: query_states, kv_states = self.qkv( [ip-26-0-148-93:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:5]: return column_linear( [ip-26-0-148-93:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: sharded_logits = self.model( [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]:Traceback (most recent call last): [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:2]: main() [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:2]: result = model(**micro_batch) [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/l[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:1]: query_states, kv_states = self.qkv( oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:2]: sharded_logits = self.model( [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:2]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-147-247:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:2]: query_states, kv_states = self.qkv( [ip-26-0-148-115:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:2]: return forward_call(*input, **kwargs) [ip-26-0-148-115:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:4]: query_states, kv_states = self.qkv( [ip-26-0-147-247:1]: return forward_call(*input, **kwargs) ing_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:2]: return column_linear( [ip-26-0-148-115:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:1]: return column_linear( [ip-26-0-147-247:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:4]: return column_linear( [ip-26-0-147-247:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:1]:Traceback (most recent call last): [ip-26-0-148-115:0]:Traceback (most recent call last): [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:0]: main() [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:0]: result = model(**micro_batch) [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:1]: main() [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:0]: sharded_logits = self.model( [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-151:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:1]: result = model(**micro_batch) [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-115:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:1]: sharded_logits = self.model( [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]:Traceback (most recent call last): [ip-26-0-148-151:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:1]: query_states, kv_states = self.qkv( [ip-26-0-148-151:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:1]: return column_linear( [ip-26-0-148-151:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:1]: main() [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:1]: result = model(**micro_batch) [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:1]: sharded_logits = self.model( [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]:Traceback (most recent call last): [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:7]: main() [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:7]: result = model(**micro_batch) [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:7]: sharded_logits = self.model( [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:1]: query_states, kv_states = self.qkv( [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:1]: return forward_call(*input, **kwargs) [ip-26-0-148-115:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:1]: return column_linear( [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-115:7]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:7]: query_states, kv_states = self.qkv( [ip-26-0-148-115:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:7]: return forward_call(*input, **kwargs) [ip-26-0-148-115:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:7]: return column_linear( [ip-26-0-148-115:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:4]:Traceback (most recent call last): [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:4]: main() [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:4]: result = model(**micro_batch) [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:4]: sharded_logits = self.model( [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:4]: query_states, kv_states = self.qkv( [ip-26-0-147-189:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:4]: return forward_call(*input, **kwargs) [ip-26-0-147-189:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:4]: return column_linear( [ip-26-0-147-189:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:3]:Traceback (most recent call last): [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:3]: main() [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:3]: result = model(**micro_batch) [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:3]: sharded_logits = self.model( [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:3]: query_states, kv_states = self.qkv( [ip-26-0-147-189:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:3]: return forward_call(*input, **kwargs) [ip-26-0-147-189:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:3]: return column_linear( [ip-26-0-147-189:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:7]:Traceback (most recent call last): [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:7]: main() [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:7]: result = model(**micro_batch) [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:7]: sharded_logits = self.model( [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:7]: query_states, kv_states = self.qkv( [ip-26-0-147-141:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:7]: return forward_call(*input, **kwargs) [ip-26-0-147-141:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:7]: return column_linear( [ip-26-0-147-141:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:3]:Traceback (most recent call last): [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:3]: main() [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:3]: result = model(**micro_batch) [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:3]: sharded_logits = self.model( [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:0]:Traceback (most recent call last): [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:0]: main() [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:0]: result = model(**micro_batch) [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:0]: sharded_logits = self.model( [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0[ip-26-0-147-233:0]:Traceback (most recent call last): [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:0]: main() [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:0]: outputs = pipeline_engine.train_batch_iter( -147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:0]: query_states, kv_states = self.qkv( [ip-26-0-147-187:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:0]: return forward_call(*input, **kwargs) [ip-26-0-147-187:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:0]: return column_linear( [ip-26-0-147-187:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:0]: result = model(**micro_batch) [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:0]: sharded_logits = self.model( [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:0]: query_states, kv_states = self.qkv( [ip-26-0-147-233:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:0]: return forward_call(*input, **kwargs) [ip-26-0-147-233:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:0]: return column_linear( [ip-26-0-147-233:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:7]:Traceback (most recent call last): [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:7]: main() [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:7]: result = model(**micro_batch) [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:7]: sharded_logits = self.model( [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:7]: query_states, kv_states = self.qkv( [ip-26-0-147-233:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:7]: return column_linear( [ip-26-0-147-204:7]:Traceback (most recent call last): [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:7]: main() [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:6]:Traceback (most recent call last): [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:6]: main() [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:7]: result = model(**micro_batch) [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:7]: sharded_logits = self.model( [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:6]: result = model(**micro_batch) [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:6]: sharded_logits = self.model( [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:7]: query_states, kv_states = self.qkv( [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:6]: query_states, kv_states = self.qkv( [ip-26-0-147-204:6]:Traceback (most recent call last): [ip-26-0-147-233:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:6]: return forward_call(*input, **kwargs) [ip-26-0-147-233:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:6]: return column_linear( [ip-26-0-147-233:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:7]: return forward_call(*input, **kwargs) [ip-26-0-149-1:5]:Traceback (most recent call last): [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:3]:Traceback (most recent call last): [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:3]: main() [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:3]: result = model(**micro_batch) [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:3]: File "/fsx/l[ip-26-0-147-204:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:3]: sharded_logits = self.model( [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-147-204:6]: main() [ip-26-0-149-1:5]: main() [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-204:7]: return column_linear( [ip-26-0-147-204:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:3]: query_states, kv_states = self.qkv( [ip-26-0-148-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:3]: return column_linear( [ip-26-0-148-245:3]:TypeError: split() miss[ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:5]: result = model(**micro_batch) [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward ing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:5]: sharded_logits = self.model( [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-149-1:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:6]: result = model(**micro_batch) [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:5]: query_states, kv_states = self.qkv( [ip-26-0-148-245:2]:Traceback (most recent call last): [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:2]: main() [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: query_states, kv_states = self.qkv( [ip-26-0-149-1:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:2]: result = model(**micro_batch) [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:2]: sharded_logits = self.model( [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:1]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:6]: sharded_logits = self.model( [ip-26-0-149-1:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:5]: return column_linear( [ip-26-0-148-245:2]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:5]:Traceback (most recent call last): [ip-26-0-149-1:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:1]: return column_linear( [ip-26-0-148-245:2]: query_states, kv_states = self.qkv( [ip-26-0-148-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:2]: return forward_call(*input, **kwargs) [ip-26-0-148-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:2]: return column_linear( [ip-26-0-148-245:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-149-1:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:5]: main() [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: result = model(**micro_batch) [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:6]: query_states, kv_states = self.qkv( [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:6]: return forward_call(*input, **kwargs) [ip-26-0-147-204:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:6]: return column_linear( [ip-26-0-147-204:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:5]: sharded_logits = self.model( [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:5]: query_states, kv_states = self.qkv( [ip-26-0-147-204:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:5]: return forward_call(*input, **kwargs) [ip-26-0-147-204:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:5]: return column_linear( [ip-26-0-147-204:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:5]:Traceback (most recent call last): [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:5]: main() [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:5]: result = model(**micro_batch) [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:5]: sharded_logits = self.model( [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:5]: query_states, kv_states = self.qkv( [ip-26-0-148-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:5]: return column_linear( [ip-26-0-148-245:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:5]: query_states, kv_states = self.qkv( [ip-26-0-147-233:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:5]: return forward_call(*input, **kwargs) [ip-26-0-147-233:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:5]: return column_linear( [ip-26-0-147-233:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:1]:Traceback (most recent call last): [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:1]: main() [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:1]: result = model(**micro_batch) [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:1]: sharded_logits = self.model( [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:1]: query_states, kv_states = self.qkv( [ip-26-0-147-233:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:1]: return column_linear( [ip-26-0-147-233:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:7]:Traceback (most recent call last): [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:3]:Traceback (most recent call last): [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:7]: main() [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:7]:Traceback (most recent call last): [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:7]: main() [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:7]: result = model(**micro_batch) [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:7]: sharded_logits = self.model( [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: main() [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:7]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:3]:Traceback (most recent call last): [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:3]: main() [ip-26-0-147-233:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:3]: result = model(**micro_batch) [ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:3]: sharded_logits = self.model( [ip-26-0-148-245:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:7]: result = model(**micro_batch) [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:7]: sharded_logits = self.model( [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:7]: hidden_encoder[ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-233:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:7]: query_states, kv_states = self.qkv( [ip-26-0-148-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:7]: return forward_call(*input, **kwargs) _states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-0-148-55:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-148-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:7]: return column_linear( [ip-26-0-148-245:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:7]: query_states, kv_states = self.qkv( [ip-26-0-148-55:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:7]: return forward_call(*input, **kwargs) [ip-26-[ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]:Traceback (most recent call last): [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:6]: main() [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:6]: result = model(**micro_batch) [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/l[ip-26-0-147-204:3]:Traceback (most recent call last): [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in 0-148-55:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:7]: return column_linear( [ip-26-0-148-55:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:3]: query_states, kv_states = self.qkv( [ip-26-0-147-233:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:3]: result = model(**micro_batch) [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:3]: sharded_logits = self.model( [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:3]: return column_linear( oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:6]: sharded_logits = self.model( [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:6]: query_states, kv_states = self.qkv( [ip-26-0-147-187:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-147-204:3]: main() [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:3]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward ing_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:6]: return column_linear( [ip-26-0-147-187:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-204:3]: result = model(**micro_batch) [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:3]: sharded_logits = self.model( [ip-26-0-149-1:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:2]:Traceback (most recent call last): [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: query_states, kv_states = self.qkv( [ip-26-0-149-1:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:4]:Traceback (most recent call last): [ip-26-0-147-204:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-149-1:3]: return column_linear( [ip-26-0-149-1:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:3]:Traceback (most recent call last): [ip-26-0-148-170:4]:Traceback (most recent call last): [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:4]: main() [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:3]: main() [ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:4]: result = model(**micro_batch) [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/br[ip-26-0-147-233:2]: main() [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:3]: query_states, kv_states = self.qkv( [ip-26-0-147-204:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:3]: return forward_call(*input, **kwargs) [ip-26-0-147-204:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:3]: return column_linear( [ip-26-0-147-233:4]: main() [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:2]:Traceback (most recent call last): [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:2]: main() [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:2]: outputs = pipeline_engine.train_batch_iter( rr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:0]:Traceback (most recent call last): [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:0]: main() [ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:2]: result = model(**micro_batch) [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:4]: sharded_logits = self.model( [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:3]: result = model(**micro_batch) [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:2]: sharded_logits = self.model( [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:3]: sharded_logits = self.model( [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_para[ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) llelism/block.py", line 135, in forward [ip-26-0-148-170:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:0]: result = model(**micro_batch) [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:0]: sharded_logits = self.model( [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-233:2]: result = model(**micro_batch) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:4]: result = model(**micro_batch) [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:2]: query_states, kv_states = self.qkv( [ip-26-0-149-1:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:2]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:0]: query_states, kv_states = self.qkv( [ip-26-0-149-1:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:2]: return column_linear( [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:0]: return forward_call(*input, **kwargs) [ip-26-0-148-55:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:0]: return column_linear( [ip-26-0-148-55:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-233:2]: sharded_logits = self.model( [ip-26-0-149-1:4]:Traceback (most recent call last): [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:4]: sharded_logits = self.model( [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: main() [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:4]: query_states, kv_states = self.qkv( [ip-26-0-148-170:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:4]: result = model(**micro_batch) [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:4]: sharded_logits = self.model( [ip-26-0-148-170:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:4]: query_states, kv_states = self.qkv( [ip-26-0-149-1:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:4]: return column_linear( [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:4]:Traceback (most recent call last): [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:4]: main() [ip-26-0-149-1:4]: return forward_call(*input, **kwargs) [ip-26-0-149-1:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:4]: return column_linear( [ip-26-0-149-1:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-233:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:3]: query_states, kv_states = self.qkv( [ip-26-0-148-170:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:4]: result = model(**micro_batch) [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:3]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:4]: sharded_logits = self.model( [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-233:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]:Traceback (most recent call last): [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:3]: return column_linear( [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:4]: query_states, kv_states = self.qkv( [ip-26-0-147-187:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:4]: return column_linear( [ip-26-0-148-170:2]:Traceback (most recent call last): [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-233:2]: query_states, kv_states = self.qkv( [ip-26-0-147-204:0]: main() [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:2]: main() [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-204:0]: result = model(**micro_batch) [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:0]: sharded_logits = self.model( [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-233:4]: query_states, kv_states = self.qkv( [ip-26-0-147-233:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-233:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-233:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-233:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-233:2]: return column_linear( [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-233:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]:Traceback (most recent call last): [ip-26-0-147-233:4]: return column_linear( [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:0]: query_states, kv_states = self.qkv( [ip-26-0-147-204:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) [ip-26-0-147-233:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:0]: return forward_call(*input, **kwargs) [ip-26-0-147-204:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:0]: return column_linear( [ip-26-0-147-204:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-233:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:2]: result = model(**micro_batch) [ip-26-0-147-204:2]:Traceback (most recent call last): [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: main() [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-149-1:0]:Traceback (most recent call last): [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-149-1:0]: main() [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-149-1:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:2]: result = model(**micro_batch) [ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:2]: sharded_logits = self.model( [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]:Traceback (most recent call last): [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:6]: main() [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:6]: result = model(**micro_batch) [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:6]: File "/fsx/l[ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-149-1:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-149-1:0]: result = model(**micro_batch) [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-149-1:0]: sharded_logits = self.model( [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-149-1:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: sharded_logits = self.model( [ip-26-0-147-204:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-149-1:0]: output = self.pp_block(**new_kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:2]: query_states, kv_states = self.qkv( [ip-26-0-149-1:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-148-170:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]: return forward_call(*input, **kwargs) [ip-26-0-147-204:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:2]: return column_linear( [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-149-1:0]: query_states, kv_states = self.qkv( [ip-26-0-149-1:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-149-1:0]: return forward_call(*input, **kwargs) [ip-26-0-149-1:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-149-1:0]: return column_linear( [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-55:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:5]: query_states, kv_states = self.qkv( [ip-26-0-148-55:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:5]: return forward_call(*input, **kwargs) [ip-26-0-148-55:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:5]: return column_linear( [ip-26-0-148-55:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-149-1:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:6]: sharded_logits = self.model( [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:6]: query_states, kv_states = self.qkv( [ip-26-0-148-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:6]: return column_linear( [ip-26-0-148-193:6]:TypeError: split() miss[ip-26-0-148-170:1]: query_states, kv_states = self.qkv( ing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:1]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]:Traceback (most recent call last): [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:6]: main() [ip-26-0-148-170:6]: main() [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:1]:Traceback (most recent call last): [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:2]:Traceback (most recent call last): [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:0]: sharded_logits = self.model( [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:0]: output = self.pp_block(**new_kwar[ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:6]: result = model(**micro_batch) [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:6]: sharded_logits = self.model( [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/[ip-26-0-148-170:1]: return column_linear( [ip-26-0-148-170:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-55:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:1]: main() [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-55:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:1]: result = model(**micro_batch) [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:2]: main() [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:6]: query_states, kv_states = self.qkv( [ip-26-0-148-55:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: sharded_logits = self.model( [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:2]: result = model(**micro_batch) [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:2]: sharded_logits = self.model( [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:6]: return forward_call(*input, **kwargs) [ip-26-0-148-55:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:6]: return column_linear( [ip-26-0-148-55:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl gs) [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:0]: query_states, kv_states = self.qkv( [ip-26-0-148-93:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:0]: return forward_call(*input, **kwargs) [ip[ip-26-0-148-170:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl -26-0-148-93:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:0]: return column_linear( [ip-26-0-148-93:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:6]: result = model(**micro_batch) [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:1]: query_states, kv_states = self.qkv( [ip-26-0-148-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:2]: query_states, kv_states = self.qkv( [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:1]: return forward_call(*input, **kwargs) [ip-26-0-148-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:1]: return column_linear( [ip-26-0-148-193:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-55:4]:Traceback (most recent call last): [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:2]: return column_linear( [ip-26-0-147-187:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-55:4]: main() [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-55:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:3]:Traceback (most recent call last): [ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-55:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-55:4]: result = model(**micro_batch) [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-55:4]: sharded_logits = self.model( [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]:Traceback (most recent call last): [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:5]: main() [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:5]: result = model(**micro_batch) [ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]:Traceback (most recent call last): [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-55:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-55:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:2]: main() [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:3]: main() [ip-26-0-148-55:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-55:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-55:4]: query_states, kv_states = self.qkv( [ip-26-0-148-55:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]: result = model(**micro_batch) [ip-26-0-147-187:7]:Traceback (most recent call last): [ip-26-0-148-55:4]: return forward_call(*input, **kwargs) [ip-26-0-148-55:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:5]: sharded_logits = self.model( [ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/minicon[ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]: sharded_logits = self.model( da3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:2]: sharded_logits = self.model( [ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:5]: query_states, kv_states = self.qkv( [ip-26-0-147-245:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:5]: return forward_call(*input, **kwargs) [ip-26-0-147-245:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:5]: return column_linear( [ip-26-0-147-245:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]:Traceback (most recent call last): [ip-26-0-148-170:2]: query_states, kv_states = self.qkv( [ip-26-0-148-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:2]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-55:4]: return column_linear( [ip-26-0-148-55:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:2]: query_states, kv_states = self.qkv( [ip-26-0-148-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:2]: return forward_call(*input, **kwargs) [ip-26-0-148-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:2]: return column_linear( [ip-26-0-148-193:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:7]: main() [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:3]: result = model(**micro_batch) [ip-26-0-148-170:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:2]: return column_linear( [ip-26-0-148-170:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:7]:Traceback (most recent call last): [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:7]: main() [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:5]: main() [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:6]: query_states, kv_states = self.qkv( [ip-26-0-148-170:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:6]: return forward_call(*input, **kwargs) [ip-26-0-148-170:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-170:6]: return column_linear( [ip-26-0-148-170:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:4]:Traceback (most recent call last): [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:6]:Traceback (most recent call last): [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:7]: result = model(**micro_batch) [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:6]: main() [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: result = model(**micro_batch) [ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:6]: result = model(**micro_batch) [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:6]: sharded_logits = self.model( [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: main() [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-193:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:7]: sharded_logits = self.model( [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:6]: query_states, kv_states = self.qkv( [ip-26-0-147-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:6]: return column_linear( [ip-26-0-147-245:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-193:4]: result = model(**micro_batch) [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:3]: sharded_logits = self.model( [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-148-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:5]: result = model(**micro_batch) [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:4]: sharded_logits = self.model( [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-193:7]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: sharded_logits = self.model( [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]:Traceback (most recent call last): [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-193:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-193:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: query_states, kv_states = self.qkv( [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:7]: main() [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-170:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-170:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-170:7]: result = model(**micro_batch) [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-170:7]: sharded_logits = self.model( [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:7]: return column_linear( [ip-26-0-148-193:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]:Traceback (most recent call last): [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:3]: main() [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-170:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-193:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:5]:Traceback (most recent call last): [ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-170:7]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-193:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-170:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-193:4]: query_states, kv_states = self.qkv( [ip-26-0-148-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:3]: result = model(**micro_batch) [ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:3]: sharded_logits = self.model( [ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]: File[ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-170:7]: query_states, kv_states = self.qkv( [ip-26-0-148-193:4]: return forward_call(*input, **kwargs) [ip-26-0-148-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-193:4]: return column_linear( [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-170:7]: return forward_call(*input, **kwargs) [ip-26-0-148-170:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-170:7]: return column_linear( [ip-26-0-148-193:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:3]: query_states, kv_states = self.qkv( [ip-26-0-148-93:3]: File "/fsx/loub[ip-26-0-148-170:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) na/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:3]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:3]: return column_linear( [ip-26-0-148-93:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:5]: sharded_logits = self.model( [ip-26-0-147-247:5]: main() [ip-26-0-147-247:0]:Traceback (most recent call last): [ip-26-0-147-247:3]:Traceback (most recent call last): [ip-26-0-148-93:1]:Traceback (most recent call last): [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:1]: main() [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-147-247:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:1]: result = model(**micro_batch) [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:0]: main() [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:1]: sharded_logits = self.model( [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) [ip-26-0-147-187:3]: query_states, kv_states = self.qkv( [ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:1]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:1]: query_states, kv_states = self.qkv( [ip-26-0-147-187:3]: return forward_call(*input, **kwargs) [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:1]: return forward_call(*input, **kwargs) [ip-26-0-148-93:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:1]: return column_linear( [ip-26-0-147-187:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-187:3]: return column_linear( [ip-26-0-147-187:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:3]: main() [ip-26-0-148-93:6]:Traceback (most recent call last): [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:6]: main() [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:5]: result = model(**micro_batch) [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:3]: result = model(**micro_batch) [ip-26-0-147-187:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: sharded_logits = self.model( [ip-26-0-148-93:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:6]: result = model(**micro_batch) [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]:Traceback (most recent call last): [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:6]: sharded_logits = self.model( [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:7]: query_states, kv_states = self.qkv( [ip-26-0-147-247:3]: sharded_logits = self.model( [ip-26-0-148-93:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]:Traceback (most recent call last): [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:7]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: main() [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:7]: return column_linear( [ip-26-0-147-187:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:0]: result = model(**micro_batch) [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:5]: query_states, kv_states = self.qkv( [ip-26-0-147-247:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:5]: return forward_call(*input, **kwargs) [ip-26-0-147-187:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:5]: return column_linear( [ip-26-0-147-187:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-147-187:1]:Traceback (most recent call last): [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:1]: main() [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-187:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-187:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-187:1]: result = model(**micro_batch) [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: main() [ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:7]: result = model(**micro_batch) [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-187:1]: sharded_logits = self.model( [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:6]: query_states, kv_states = self.qkv( [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-187:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-187:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-187:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-187:1]: query_states, kv_states = self.qkv( [ip-26-0-147-187:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-187:1]: return forward_call(*input, **kwargs) [ip-26-0-147-187:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-187:1]: return column_linear( [ip-26-0-147-187:1]:TypeError: split() miss[ip-26-0-147-247:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward ing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:6]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: sharded_logits = self.model( [ip-26-0-147-247:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: sharded_logits = self.model( [ip-26-0-148-93:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-148-93:6]: return column_linear( [ip-26-0-148-93:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:7]: query_states, kv_states = self.qkv( [ip-26-0-148-93:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:7]: return forward_call(*input, **kwargs) [ip-26-0-148-93:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:7]: return column_linear( [ip-26-0-148-93:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:2]:Traceback (most recent call last): [ip-26-0-148-151:6]:Traceback (most recent call last): [ip-26-0-147-247:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:3]: query_states, kv_states = self.qkv( [ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:2]: result = model(**micro_batch) [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:2]: main() [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-151:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:6]: main() [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:2]: sharded_logits = self.model( [ip-26-0-148-151:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]:Traceback (most recent call last): [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:6]: main() [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:6]: result = model(**micro_batch) [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/l[ip-26-0-148-151:2]: result = model(**micro_batch) [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:6]: result = model(**micro_batch) [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:2]: sharded_logits = self.model( [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]:Traceback (most recent call last): [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: main() [ip-26-0-147-247:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:6]: sharded_logits = self.model( [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-148-151:6]: sharded_logits = self.model( [ip-26-0-147-247:2]: output = self.pp_block(**new_kwargs) ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:6]: query_states, kv_states = self.qkv( [ip-26-0-148-115:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:6]: return forward_call(*input, **kwargs) [ip-26-0-148-115:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ing_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:6]: return column_linear( [ip-26-0-148-115:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:0]: query_states, kv_states = self.qkv( [ip-26-0-148-115:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:0]: return forward_call(*input, **kwargs) [ip-26-0-148-115:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:0]: ret[ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:3]: return forward_call(*input, **kwargs) urn column_linear( [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:3]: return column_linear( [ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-115:4]:Traceback (most recent call last): [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: main() [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:5]: query_states, kv_states = self.qkv( [ip-26-0-147-245:4]:Traceback (most recent call last): [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:4]: main() [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:4]: result = model(**micro_batch) [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/l[ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:4]: result = model(**micro_batch) [ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:4]: sharded_logits = self.model( [ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:5]: return column_linear( [ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:4]: result = model(**micro_batch) [ip-26-0-147-247:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) oubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:4]: sharded_logits = self.model( [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/s[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:4]: query_states, kv_states = self.qkv( [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ite-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:4]: query_states, kv_states = self.qkv( [ip-26-0-147-245:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:4]: return forward_call(*input, **kwargs) [ip-26-0-147-245:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/model[ip-26-0-148-115:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:4]: return forward_call(*input, **kwargs) [ip-26-0-148-115:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:4]: return column_linear( [ip-26-0-148-115:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) ing_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:4]: return column_linear( [ip-26-0-147-245:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]:Traceback (most recent call last): [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:2]: main() [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:7]:Traceback (most recent call last): [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:2]: result = model(**micro_batch) [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:7]: main() [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:2]: sharded_logits = self.model( [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:2]: [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:4]: sharded_logits = self.model( [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:2]: query_states, kv_states = self.qkv( query_states, kv_states = self.qkv( [ip-26-0-148-151:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:2]: return forward_call(*input, **kwargs) [ip-26-0-147-245:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:2]: return column_linear( [ip-26-0-147-245:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:1]:Traceback (most recent call last): [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:1]: main() [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:0]: query_states, kv_states = self.qkv( [ip-26-0-147-247:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:2]: return column_linear( [ip-26-0-148-151:6]: query_states, kv_states = self.qkv( [ip-26-0-147-247:0]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:0]: return column_linear( [ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:6]: return forward_call(*input, **kwargs) [ip-26-0-148-151:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:1]: result = model(**micro_batch) [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:1]: sharded_logits = self.model( [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:6]: return column_linear( [ip-26-0-147-247:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-245:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:7]: result = model(**micro_batch) [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]:Traceback (most recent call last): [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:1]: query_states, kv_states = self.qkv( [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:7]: sharded_logits = self.model( [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:1]: return column_linear( [ip-26-0-147-245:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-147-247:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:3]: main() [ip-26-0-147-247:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:7]: query_states, kv_states = self.qkv( [ip-26-0-147-247:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:7]: return column_linear( [ip-26-0-147-247:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:3]: result = model(**micro_batch) [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:2]: query_states, kv_states = self.qkv( [ip-26-0-148-151:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:3]:Traceback (most recent call last): [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:0]:Traceback (most recent call last): [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:0]: main() [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:3]: main() [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:3]: result = model(**micro_batch) [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:3]: sharded_logits = self.model( [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:0]: result = model(**micro_batch) [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:1]:Traceback (most recent call last): [ip-26-0-147-245:0]: sharded_logits = self.model( [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:3]: query_states, kv_states = self.qkv( [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:3]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:3]: return column_linear( [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-115:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:0]: query_states, kv_states = self.qkv( [ip-26-0-147-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:5]:Traceback (most recent call last): [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:5]: main() [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-245:0]: return column_linear( [ip-26-0-147-245:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:1]: main() [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-193:0]:Traceback (most recent call last): [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:0]: main() [ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:5]: result = model(**micro_batch) [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:5]: sharded_logits = self.model( [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-193:6]:Traceback (most recent call last): [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:6]: main() [ip-26-0-147-193:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-193:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:6]: result = model(**micro_batch) [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:7]:Traceback (most recent call last): [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:4]:Traceback (most recent call last): [ip-26-0-147-193:3]:Traceback (most recent call last): [ip-26-0-147-245:7]: main() [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:7]: result = model(**micro_batch) [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:6]: sharded_logits = self.model( [ip-26-0-147-245:3]:Traceback (most recent call last): [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:0]: result = model(**micro_batch) [ip-26-0-147-245:7]: sharded_logits = self.model( [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:3]: main() [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:1]: result = model(**micro_batch) [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:2]:Traceback (most recent call last): [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-245:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:2]: main() [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:3]: result = model(**micro_batch) [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:0]: sharded_logits = self.model( [ip-26-0-147-245:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-245:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:3]: sharded_logits = self.model( [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-193:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: main() [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:3]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:7]: query_states, kv_states = self.qkv( [ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: main() [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-245:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:7]: return forward_call(*input, **kwargs) [ip-26-0-147-245:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:7]: return column_linear( [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-245:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-245:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-245:3]: query_states, kv_states = self.qkv( [ip-26-0-147-245:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-245:3]: return forward_call(*input, **kwargs) [ip-26-0-147-245:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:4]: result = model(**micro_batch) [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-245:3]: return column_linear( [ip-26-0-147-245:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:6]: query_states, kv_states = self.qkv( [ip-26-0-147-193:3]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:3]: result = model(**micro_batch) [ip-26-0-147-193:1]: sharded_logits = self.model( [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:4]: sharded_logits = self.model( [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]:Traceback (most recent call last): [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-193:7]: main() [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-193:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:2]: result = model(**micro_batch) [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:4]: query_states, kv_states = self.qkv( [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:0]: query_states, kv_states = self.qkv( [ip-26-0-147-193:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-193:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-193:7]: result = model(**micro_batch) [ip-26-0-147-193:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:0]: return forward_call(*input, **kwargs) [ip-26-0-147-193:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:0]: return column_linear( [ip-26-0-147-193:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:4]: return forward_call(*input, **kwargs) [ip-26-0-147-193:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:3]: sharded_logits = self.model( [ip-26-0-147-193:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:4]: return column_linear( [ip-26-0-147-193:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:2]: sharded_logits = self.model( [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:6]: return column_linear( [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-193:7]: sharded_logits = self.model( [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:3]: query_states, kv_states = self.qkv( [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-193:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:3]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:3]: return column_linear( [ip-26-0-147-193:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:1]: query_states, kv_states = self.qkv( [ip-26-0-147-193:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:5]: query_states, kv_states = self.qkv( [ip-26-0-147-193:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:5]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-193:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:1]: return forward_call(*input, **kwargs) [ip-26-0-147-193:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:1]: return column_linear( [ip-26-0-147-193:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:5]: return column_linear( [ip-26-0-147-193:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-193:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:7]: query_states, kv_states = self.qkv( [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-193:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: query_states, kv_states = self.qkv( [ip-26-0-147-193:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-193:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:7]: return column_linear( [ip-26-0-147-193:2]: return forward_call(*input, **kwargs) [ip-26-0-147-193:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-193:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-193:2]: return column_linear( [ip-26-0-147-193:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:0]:Traceback (most recent call last): [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:0]: main() [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:5]:Traceback (most recent call last): [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:5]: main() [ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:0]: result = model(**micro_batch) [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:0]: sharded_logits = self.model( [ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:5]: result = model(**micro_batch) [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]:Traceback (most recent call last): [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:5]: sharded_logits = self.model( [ip-26-0-147-189:2]: main() [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:2]: result = model(**micro_batch) [ip-26-0-147-189:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:0]: query_states, kv_states = self.qkv( [ip-26-0-147-189:2]: sharded_logits = self.model( [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:0]: return forward_call(*input, **kwargs) [ip-26-0-147-189:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:0]: return column_linear( [ip-26-0-147-189:5]: query_states, kv_states = self.qkv( [ip-26-0-147-189:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:5]: return forward_call(*input, **kwargs) [ip-26-0-147-189:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:5]: return column_linear( [ip-26-0-147-189:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:2]: query_states, kv_states = self.qkv( [ip-26-0-147-189:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:2]: return forward_call(*input, **kwargs) [ip-26-0-147-189:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:2]: return column_linear( [ip-26-0-147-189:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-115:5]:Traceback (most recent call last): [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-115:5]: main() [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-115:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-115:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-115:5]: result = model(**micro_batch) [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-115:5]: sharded_logits = self.model( [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-115:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-115:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-115:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-115:5]: query_states, kv_states = self.qkv( [ip-26-0-148-115:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-115:5]: return forward_call(*input, **kwargs) [ip-26-0-148-115:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-115:5]: return column_linear( [ip-26-0-148-115:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:6]:Traceback (most recent call last): [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:6]: main() [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:6]: result = model(**micro_batch) [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:6]: sharded_logits = self.model( [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:6]: query_states, kv_states = self.qkv( [ip-26-0-147-189:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:6]: return forward_call(*input, **kwargs) [ip-26-0-147-189:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:6]: return column_linear( [ip-26-0-147-189:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:4]:Traceback (most recent call last): [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:4]: main() [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:4]: result = model(**micro_batch) [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:4]: sharded_logits = self.model( [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:4]: query_states, kv_states = self.qkv( [ip-26-0-147-141:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:4]: return forward_call(*input, **kwargs) [ip-26-0-147-141:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:4]: return column_linear( [ip-26-0-147-141:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:1]:Traceback (most recent call last): [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:1]: main() [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:1]: result = model(**micro_batch) [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:1]: sharded_logits = self.model( [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:1]: query_states, kv_states = self.qkv( [ip-26-0-147-189:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:1]: return forward_call(*input, **kwargs) [ip-26-0-147-189:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:1]: return column_linear( [ip-26-0-147-189:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-189:7]:Traceback (most recent call last): [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-189:7]: main() [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-189:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-189:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-189:7]: result = model(**micro_batch) [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-189:7]: sharded_logits = self.model( [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-189:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-189:7]: output = self.pp_block(**new_kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-189:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-189:7]: query_states, kv_states = self.qkv( [ip-26-0-147-189:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-189:7]: return forward_call(*input, **kwargs) [ip-26-0-147-189:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-189:7]: return column_linear( [ip-26-0-147-189:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:2]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]:Traceback (most recent call last): [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:2]: main() [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:4]: query_states, kv_states = self.qkv( [ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:2]: result = model(**micro_batch) [ip-26-0-148-151:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:2]: sharded_logits = self.model( [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:4]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-151:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:2]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:2]: query_states, kv_states = self.qkv( [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-93:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:2]: return forward_call(*input, **kwargs) [ip-26-0-148-93:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:2]: return column_linear( [ip-26-0-148-151:2]: return column_linear( [ip-26-0-148-93:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:4]: return column_linear( [ip-26-0-148-151:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-93:4]:Traceback (most recent call last): [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-93:4]: main() [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-93:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-93:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-93:4]: result = model(**micro_batch) [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: return forward_call(*input, **kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/code/f[ip-26-0-148-151:3]: sharded_logits = self.model( [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl ork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-93:4]: sharded_logits = self.model( [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: return forward_call(*input, **kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-93:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: return forward_call(*input, **kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-93:4]: output = self.pp_block(**new_kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: retu[ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward rn forward_call(*input, **kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-93:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: return forward_call(*input, **kwargs) [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-93:4]: query_states, kv_states = self.qkv( [ip-26-0-148-93:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-93:4]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-93:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-93:4]: return column_linear( [ip-26-0-148-93:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:3]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:3]: query_states, kv_states = self.qkv( [ip-26-0-148-151:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:3]: return forward_call(*input, **kwargs) [ip-26-0-148-151:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:3]: return column_linear( [ip-26-0-148-151:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:7]:Traceback (most recent call last): [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:7]: main() [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-151:7]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:7]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:7]: result = model(**micro_batch) [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:7]: sharded_logits = self.model( [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:7]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:7]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:7]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:7]: query_states, kv_states = self.qkv( [ip-26-0-148-151:7]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:7]: return forward_call(*input, **kwargs) [ip-26-0-148-151:7]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:7]: return column_linear( [ip-26-0-148-151:7]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:2]:Traceback (most recent call last): [ip-26-0-147-141:0]:Traceback (most recent call last): [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:2]: main() [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:0]: main() [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:2]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:2]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:2]: result = model(**micro_batch) [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: result = model(**micro_batch) [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:2]: sharded_logits = self.model( [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:0]: sharded_logits = self.model( [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:2]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:2]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:6]:Traceback (most recent call last): [ip-26-0-147-141:2]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]:Traceback (most recent call last): [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:5]: main() [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:2]: query_states, kv_states = self.qkv( [ip-26-0-147-141:2]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:2]: return forward_call(*input, **kwargs) [ip-26-0-147-141:2]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:2]: return column_linear( [ip-26-0-147-141:2]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:3]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:5]: result = model(**micro_batch) [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:5]: sharded_logits = self.model( [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:3]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:5]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:0]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:6]: main() [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:3]: query_states, kv_states = self.qkv( [ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:3]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]:Traceback (most recent call last): [ip-26-0-147-141:6]: result = model(**micro_batch) [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:3]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:3]: return column_linear( [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:6]: sharded_logits = self.model( [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:3]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:5]: query_states, kv_states = self.qkv( [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-141:1]: main() [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-141:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-141:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-141:1]: result = model(**micro_batch) [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-141:1]: sharded_logits = self.model( [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return forward_call(*input, **kwargs) [ip-26-0-147-141:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:5]: return column_linear( [ip-26-0-147-141:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:0]: query_states, kv_states = self.qkv( [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-141:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-141:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:6]: query_states, kv_states = self.qkv( [ip-26-0-147-141:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:6]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-141:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:0]: return column_linear( [ip-26-0-147-141:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-141:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:6]: return column_linear( [ip-26-0-147-141:1]: query_states, kv_states = self.qkv( [ip-26-0-147-141:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-141:1]: return forward_call(*input, **kwargs) [ip-26-0-147-141:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-141:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-141:1]: return column_linear( [ip-26-0-147-141:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:0]:Traceback (most recent call last): [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:0]: main() [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:0]: result = model(**micro_batch) [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:0]: sharded_logits = self.model( [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:0]: query_states, kv_states = self.qkv( [ip-26-0-148-245:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:0]: return forward_call(*input, **kwargs) [ip-26-0-148-245:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:0]: return column_linear( [ip-26-0-148-245:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:6]:Traceback (most recent call last): [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-245:6]: main() [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-245:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-245:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-245:6]: result = model(**micro_batch) [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-245:6]: sharded_logits = self.model( [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-245:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-245:6]: output = self.pp_block(**new_kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-245:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-245:6]: query_states, kv_states = self.qkv( [ip-26-0-148-245:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-245:6]: return forward_call(*input, **kwargs) [ip-26-0-148-245:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-245:6]: return column_linear( [ip-26-0-148-245:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-247:6]:Traceback (most recent call last): [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-247:6]: main() [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-247:6]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-247:6]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-247:6]: result = model(**micro_batch) [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-247:6]: sharded_logits = self.model( [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-247:6]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-247:6]: output = self.pp_block(**new_kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-247:6]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-247:6]: query_states, kv_states = self.qkv( [ip-26-0-147-247:6]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-247:6]: return forward_call(*input, **kwargs) [ip-26-0-147-247:6]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-247:6]: return column_linear( [ip-26-0-147-247:6]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:5]:Traceback (most recent call last): [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:5]: main() [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-151:5]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:5]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:5]: result = model(**micro_batch) [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:5]: sharded_logits = self.model( [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:5]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:5]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:5]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:5]: query_states, kv_states = self.qkv( [ip-26-0-148-151:0]:Traceback (most recent call last): [ip-26-0-148-151:5]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:5]: return forward_call(*input, **kwargs) [ip-26-0-148-151:5]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:5]: return column_linear( [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-148-151:0]: main() [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-148-151:5]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-151:0]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-148-151:0]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-148-151:0]: result = model(**micro_batch) [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-148-151:0]: sharded_logits = self.model( [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-148-151:0]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-148-151:0]: output = self.pp_block(**new_kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-148-151:0]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-148-151:0]: query_states, kv_states = self.qkv( [ip-26-0-148-151:0]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-148-151:0]: return forward_call(*input, **kwargs) [ip-26-0-148-151:0]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-148-151:0]: return column_linear( [ip-26-0-148-151:0]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:4]:Traceback (most recent call last): [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:4]: main() [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:4]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:4]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-204:4]: result = model(**micro_batch) [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:4]: sharded_logits = self.model( [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:4]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:4]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:4]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:4]: query_states, kv_states = self.qkv( [ip-26-0-147-204:4]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:4]: return forward_call(*input, **kwargs) [ip-26-0-147-204:4]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:4]: return column_linear( [ip-26-0-147-204:4]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-147-204:1]:Traceback (most recent call last): [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1592, in [ip-26-0-147-204:1]: main() [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/main.py", line 1150, in main [ip-26-0-147-204:1]: outputs = pipeline_engine.train_batch_iter( [ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 210, in train_batch_iter [ip-26-0-147-204:1]: output = self.forward(context=context, state=state, micro_batch=micro_batch, model=model) [ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/engine.py", line 43, in forward [ip-26-0-147-204:1]: result = model(**micro_batch) [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 751, in forward [ip-26-0-147-204:1]: sharded_logits = self.model( [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 681, in forward [ip-26-0-147-204:1]: hidden_encoder_states = encoder_block(**hidden_encoder_states) [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/brrr/brrr/parallelism/pipeline_parallelism/block.py", line 135, in forward [ip-26-0-147-204:1]: output = self.pp_block(**new_kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 500, in forward [ip-26-0-147-204:1]: output = self.attn(hidden_states=hidden_states, sequence_mask=sequence_mask) [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 409, in forward [ip-26-0-147-204:1]: query_states, kv_states = self.qkv( [ip-26-0-147-204:1]: File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1194, in _call_impl [ip-26-0-147-204:1]: return forward_call(*input, **kwargs) [ip-26-0-147-204:1]: File "/fsx/loubna/code/fork/brrr/examples/gpt2_mqa/modeling_gpt2_fast.py", line 281, in forward [ip-26-0-147-204:1]: return column_linear( [ip-26-0-147-204:1]:TypeError: split() missing required argument 'split_size_or_sections' (pos 2) [ip-26-0-148-245:7]:libfabric:350439:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-147-204:3]:libfabric:3476105:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-148-115:7]:libfabric:2603751:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-147-141:3]:libfabric:3682184:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-147-204:7]:libfabric:3476109:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-148-115:5]:libfabric:2603749:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-147-204:5]:libfabric:3476107:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-148-245:3]:libfabric:350435:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight [ip-26-0-148-245:5]:libfabric:350437:1686832261::efa:ep_ctrl:efa_rdm_peer_clear():123 Closing EP with unacked CONNREQs in flight ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 350432) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 351210) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3236869) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 6919) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 4105682) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3476102) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2494830) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3257735) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3975165) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2603744) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 2876006) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3655527) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 150955) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3609392) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 842377) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: 1) local_rank: 0 (pid: 3682181) of binary: /fsx/loubna/miniconda3/envs/megatron/bin/python Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in return _run_code(code, main_globals, None, Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main return _run_code(code, main_globals, None, return _run_code(code, main_globals, None, return _run_code(code, main_globals, None, Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main exec(code, run_globals) return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main exec(code, run_globals) exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main main() return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main exec(code, run_globals) return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code return _run_code(code, main_globals, None, return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code return _run_code(code, main_globals, None, return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run return _run_code(code, main_globals, None, run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in exec(code, run_globals) return f(*args, **kwargs) return _run_code(code, main_globals, None, exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code exec(code, run_globals) return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run return f(*args, **kwargs) elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main run(args) main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main return f(*args, **kwargs) elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ run(args) main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper raise ChildFailedError( elastic_launch( raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 25 (local_rank: 1) exitcode : 1 (pid: 3609393) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 26 (local_rank: 2) exitcode : 1 (pid: 3609394) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 27 (local_rank: 3) exitcode : 1 (pid: 3609395) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 28 (local_rank: 4) exitcode : 1 (pid: 3609396) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 29 (local_rank: 5) exitcode : 1 (pid: 3609397) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 30 (local_rank: 6) exitcode : 1 (pid: 3609398) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 31 (local_rank: 7) exitcode : 1 (pid: 3609399) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-193.us-west-2.compute.internal rank : 24 (local_rank: 0) exitcode : 1 (pid: 3609392) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ main() torch.distributed.elastic.multiprocessing.errors File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent return f(*args, **kwargs) return launch_agent(self._config, self._entrypoint, list(args)) .ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 113 (local_rank: 1) exitcode : 1 (pid: 3975166) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 114 (local_rank: 2) exitcode : 1 (pid: 3975167) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 115 (local_rank: 3) exitcode : 1 (pid: 3975168) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 hos run(args) run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run elastic_launch( t : ip-26-0-148-93.us-west-2.compute.internal rank : 116 (local_rank: 4) exitcode : 1 (pid: 3975169) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 117 (local_rank: 5) exitcode : 1 (pid: 3975170) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 118 (local_rank: 6) exitcode : 1 (pid: 3975171) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 119 (local_rank: 7) exitcode : 1 (pid: 3975172) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-93.us-west-2.compute.internal rank : 112 (local_rank: 0) exitcode : 1 (pid: 3975165) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main raise ChildFailedError( return f(*args, **kwargs) return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 1 (local_rank: 1) exitcode : 1 (pid: 3682182) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 2 (local_rank: 2) exitcode : 1 (pid: 3682183) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 3 (local_rank: 3) exitcode : 1 (pid: 3682184) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent : ip-26-0-147-141.us-west-2.compute.internal rank : 4 (local_rank: 4) exitcode : 1 (pid: 3682185) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 5 (local_rank: 5) exitcode : 1 (pid: 3682186) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 6 (local_rank: 6) exitcode : 1 (pid: 3682187) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 7 (local_rank: 7) exitcode : 1 (pid: 3682188) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-141.us-west-2.compute.internal rank : 0 (local_rank: 0) exitcode : 1 (pid: 3682181) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main raise ChildFailedError( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors. return f(*args, **kwargs) elastic_launch( ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 65 (local_rank: 1) exitcode : 1 (pid: 2603745) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 66 (local_rank: 2) exitcode : 1 (pid: 2603746) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 67 (local_rank: 3) exitcode : 1 (pid: 2603747) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 hosttorch.distributed.elastic.multiprocessing.errors.ChildFailedError: : ip-26-0-148-115.us-west-2.compute.internal rank : 68 (local_rank: 4) exitcode : 1 (pid: 2603748) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 69 (local_rank: 5) exitcode : 1 (pid: 2603749) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 70 (local_rank: 6) exitcode : 1 (pid: 2603750) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 71 (local_rank: 7) exitcode : 1 (pid: 2603751) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elast File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 49 (local_rank: 1) exitcode : 1 (pid: 3655528) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 50 (local_rank: 2) exitcode : 1 (pid: 3655529) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 51 (local_rank: 3) exitcode : 1 (pid: 3655530) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-14ic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-115.us-west-2.compute.internal rank : 64 (local_rank: 0) exitcode : 1 (pid: 2603744) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main raise ChildFailedError( 7-245.us-west-2.compute.internal rank : 52 (local_rank: 4) exitcode : 1 (pid: 3655531) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 53 (local_rank: 5) exitcode : 1 (pid: 3655532) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 54 (local_rank: 6) exitcode : 1 (pid: 3655533) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 55 (local_rank: 7) exitcode : 1 (pid: 3655534) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-245.us-west-2.compute.internal rank : 48 (local_rank: 0) exitcode : 1 (pid: 3655527) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 97 (local_rank: 1) exitcode : 1 (pid: 350433) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 98 (local_rank: 2) exitcode : 1 (pid: 350434) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 99 (local_rank: 3) exitcode : 1 (pid: 350435) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 100 (local_rank: 4) exitcode : 1 (pid: 350436) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 101 (local_rank: 5) exitcode : 1 (pid: 350437) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 102 (local_rank: 6) exitcode : 1 (pid: 350438) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 103 (local_rank: 7) exitcode : 1 (pid: 350439) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-245.us-west-2.compute.internal rank : 96 (local_rank: 0) exitcode : 1 (pid: 350432) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ return f(*args, **kwargs) return launch_agent(self._config, self._entrypoint, list(args)) return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run run(args) run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run raise ChildFailedError( raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 81 (local_rank: 1) exitcode : 1 (pid: 6920) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 82 (local_rank: 2) exitcode : 1 (pid: 6921) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 83 (local_rank: 3) exitcode : 1 (pid: 6922) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : itorch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 9 (local_rank: 1) exitcode : 1 (pid: 2494831) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 10 (local_rank: 2) exitcode : 1 (pid: 2494832) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 11 (local_rank: 3) exitcode : 1 (pid: 2494833) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.htp-26-0-148-170.us-west-2.compute.internal rank : 84 (local_rank: 4) exitcode : 1 (pid: 6923) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 85 (local_rank: 5) exitcode : 1 (pid: 6924) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 86 (local_rank: 6) exitcode : 1 (pid: 6925) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 87 (local_rank: 7) exitcode : 1 (pid: 6926) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------ml [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 12 (local_rank: 4) exitcode : 1 (pid: 2494834) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 13 (local_rank: 5) exitcode : 1 (pid: 2494835) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 14 (local_rank: 6) exitcode : 1 (pid: 2494836) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 15 (local_rank: 7) exitcode : 1 (pid: 2494837) error_file: traceback : To enable trac------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-170.us-west-2.compute.internal rank : 80 (local_rank: 0) exitcode : 1 (pid: 6919) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ eback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-187.us-west-2.compute.internal rank : 8 (local_rank: 0) exitcode : 1 (pid: 2494830) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ elastic_launch( main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent return f(*args, **kwargs) Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent run(args) elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 121 (local_rank: 1) exitcode : 1 (pid: 351211) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 122 (local_rank: 2) exitcode : 1 (pid: 351212) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 123 (local_rank: 3) exitcode : 1 (pid: 351213) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 124 (local_rank: 4) exitcode : 1 (pid: 351214) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 125 (local_rank: 5) exitcode : 1 (pid: 351215) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 126 (local_rank: 6) exitcode : 1 (pid: 351216) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 127 (local_rank: 7) exitcode : 1 (pid: 351217) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ----------------- raise ChildFailedError( return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code ------------------------------------------- Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-149-1.us-west-2.compute.internal rank : 120 (local_rank: 0) exitcode : 1 (pid: 351210) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 89 (local_rank: 1) exitcode : 1 (pid: 3257736) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 90 (local_rank: 2) exitcode : 1 (pid: 3257737) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 91 (local_rank: 3) exitcode : 1 (pid: 3257738) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 92 (local_rank: 4) exitcode : 1 (pid: 3257739) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 93 (local_rank: 5) exitcode : 1 (pid: 3257740) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 94 (local_rank: 6) exitcode : 1 (pid: 3257741) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 95 (local_rank: 7) exitcode : 1 (pid: 3257742) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elasttorch.distributed.elastic.multiprocessing.errors.ChildFailedErroric/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-193.us-west-2.compute.internal rank : 88 (local_rank: 0) exitcode : 1 (pid: 3257735) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ raise ChildFailedError( : ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 33 (local_rank: 1) exitcode : 1 (pid: 3476103) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 34 (local_rank: 2) exitcode : 1 (pid: 3476104) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 35 (local_rank: 3) exitcode : 1 (pid: 3476105) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 36 (local_rank: 4) exitcode : 1 (pid: 3476106) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 37 (local_rank: 5) exitcode : 1 (pid: 3476107) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 38 (local_rank: 6) exitcode : 1 (pid: 3476108) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 39 (local_rank: 7) exitcode : 1 (pid: 3476109) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html - exec(code, run_globals) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in ----------------------------------------------------------- Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-204.us-west-2.compute.internal rank : 32 (local_rank: 0) exitcode : 1 (pid: 3476102) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent elastic_launch( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 57 (local_rank: 1) exitcode : 1 (pid: 150956) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 58 (local_rank: 2) exitcode : 1 (pid: 150957) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 59 (local_rank: 3) exitcode : 1 (pid: 150958) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 60 (local_rank: 4) exitcode : 1 (pid: 150959) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 61 (local_rank: 5) exitcode : 1 (pid: 150960) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 62 (local_rank: 6) exitcode : 1 (pid: 150961) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 63 (local_rank: 7) exitcode : 1 (pid: 150962) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/erro File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ rs.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-247.us-west-2.compute.internal rank : 56 (local_rank: 0) exitcode : 1 (pid: 150955) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper return launch_agent(self._config, self._entrypoint, list(args)) raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent .ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 17 (local_rank: 1) exitcode : 1 (pid: 842378) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 18 (local_rank: 2) exitcode : 1 (pid: 842379) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 19 (local_rank: 3) exitcode : 1 (pid: 842380) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 20 (local_rank: 4) exitcode : 1 (pid: 842381) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 21 (local_rank: 5) exitcode : 1 (pid: 842382) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 22 (local_rank: 6) exitcode : 1 (pid: 842383) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 23 (local_rank: 7) exitcode : 1 (pid: 842384) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-189.us-west-2.compute.internal rank : 16 (local_rank: 0) exitcode : 1 (pid: 842377) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ raise ChildFailedError( return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 73 (local_rank: 1) exitcode : 1 (pid: 2876007) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 74 (local_rank: 2) exitcode : 1 (pid: 2876008) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 75 (local_rank: 3) exitcode : 1 (pid: 2876009) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 76 (local_rank: 4) exitcode : 1 (pid: 2876010) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 77 (local_rank: 5) exitcode : 1 (pid: 2876011) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 78 (local_rank: 6) exitcode : 1 (pid: 2876012) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 79 (local_rank: 7) exitcode : 1 (pid: 2876013) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-151.us-west-2.compute.internal rank : 72 (local_rank: 0) exitcode : 1 (pid: 2876006) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run Traceback (most recent call last): File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 196, in _run_module_as_main elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ return _run_code(code, main_globals, None, File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/runpy.py", line 86, in _run_code return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent exec(code, run_globals) raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 105 (local_rank: 1) exitcode : 1 (pid: 4105683) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 106 (local_rank: 2) exitcode : 1 (pid: 4105684) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 107 (local_rank: 3) exitcode : 1 (pid: 4105685) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 108 (local_rank: 4) exitcode : 1 (pid: 4105686) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 109 (local_rank: 5) exitcode : 1 (pid: 4105687) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 110 (local_rank: 6) exitcode : 1 (pid: 4105688) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 111 (local_rank: 7) exitcode : 1 (pid: 4105689) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html --- File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 766, in --------------------------------------------------------- Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-148-55.us-west-2.compute.internal rank : 104 (local_rank: 0) exitcode : 1 (pid: 4105682) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ main() File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper return f(*args, **kwargs) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main run(args) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run elastic_launch( File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) File "/fsx/loubna/miniconda3/envs/megatron/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ main.py FAILED ------------------------------------------------------------ Failures: [1]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 41 (local_rank: 1) exitcode : 1 (pid: 3236870) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [2]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 42 (local_rank: 2) exitcode : 1 (pid: 3236871) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [3]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 43 (local_rank: 3) exitcode : 1 (pid: 3236872) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [4]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 44 (local_rank: 4) exitcode : 1 (pid: 3236873) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [5]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 45 (local_rank: 5) exitcode : 1 (pid: 3236874) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [6]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 46 (local_rank: 6) exitcode : 1 (pid: 3236875) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html [7]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 47 (local_rank: 7) exitcode : 1 (pid: 3236876) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2023-06-15_12:31:03 host : ip-26-0-147-233.us-west-2.compute.internal rank : 40 (local_rank: 0) exitcode : 1 (pid: 3236869) error_file: traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ srun: error: ip-26-0-147-204: task 4: Exited with exit code 1 srun: launch/slurm: _step_signal: Terminating StepId=158050.0 srun: error: ip-26-0-147-189: task 2: Exited with exit code 1 srun: error: ip-26-0-148-193: task 13: Exited with exit code 1 srun: error: ip-26-0-149-1: task 15: Exited with exit code 1 srun: error: ip-26-0-147-247: task 7: Exited with exit code 1 srun: error: ip-26-0-147-233: task 5: Exited with exit code 1 slurmstepd: error: *** STEP 158050.0 ON ip-26-0-147-141 CANCELLED AT 2023-06-15T12:31:04 *** srun: error: ip-26-0-147-187: task 1: Terminated srun: error: ip-26-0-147-141: task 0: Terminated srun: error: ip-26-0-148-151: task 11: Terminated srun: error: ip-26-0-147-245: task 6: Terminated srun: error: ip-26-0-147-193: task 3: Terminated srun: error: ip-26-0-148-93: task 9: Terminated srun: error: ip-26-0-148-55: task 8: Terminated srun: error: ip-26-0-148-115: task 10: Terminated srun: error: ip-26-0-148-170: task 12: Terminated srun: error: ip-26-0-148-245: task 14: Terminated srun: Force Terminated StepId=158050.0