diff --git "a/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/stage3.ipynb" "b/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/stage3.ipynb" new file mode 100644--- /dev/null +++ "b/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/stage3.ipynb" @@ -0,0 +1,4581 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "2d294c56", + "metadata": { + "papermill": { + "duration": 0.005601, + "end_time": "2023-09-14T00:22:48.660046", + "exception": false, + "start_time": "2023-09-14T00:22:48.654445", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# RWKV v5\n", + "\n", + "Simple memory training for a small model\n", + "\n", + "**Note:** This project assumes you have the rwkv-infctx conda env setup" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "d8e1b078", + "metadata": { + "papermill": { + "duration": 0.003053, + "end_time": "2023-09-14T00:22:48.666813", + "exception": false, + "start_time": "2023-09-14T00:22:48.663760", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Basic Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "25b2c35a", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:22:48.675167Z", + "iopub.status.busy": "2023-09-14T00:22:48.674642Z", + "iopub.status.idle": "2023-09-14T00:22:49.678806Z", + "shell.execute_reply": "2023-09-14T00:22:49.677875Z" + }, + "papermill": { + "duration": 1.011151, + "end_time": "2023-09-14T00:22:49.681331", + "exception": false, + "start_time": "2023-09-14T00:22:48.670180", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CITATION.cff RWKV-v4wavenet\t RWKV-v5headsize2x checkpoint\tnotebook\r\n", + "LICENSE RWKV-v5\t\t RWKV-v5headsize32 datapath\toutput\r\n", + "README.md RWKV-v5-beta2\t RWKV-v5rstack\t docker\r\n", + "RWKV-v4neo RWKV-v5altwavenet RWKV-v5wavenet model\r\n" + ] + } + ], + "source": [ + "# First lets setup the various directories, and init the model\n", + "!ls ../../../../../\n", + "!mkdir -p ../../../../../model/\n", + "!mkdir -p ../../../../../datapath/\n", + "!mkdir -p ../../../../../checkpoint/" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "10fc7c1c", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:22:49.690536Z", + "iopub.status.busy": "2023-09-14T00:22:49.689989Z", + "iopub.status.idle": "2023-09-14T00:22:53.079072Z", + "shell.execute_reply": "2023-09-14T00:22:53.078184Z" + }, + "papermill": { + "duration": 3.39647, + "end_time": "2023-09-14T00:22:53.081568", + "exception": false, + "start_time": "2023-09-14T00:22:49.685098", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n", + "\u001b[0m" + ] + } + ], + "source": [ + "# Additional dependencies for eval stuff\n", + "!pip3 install -q aiocsv aiofiles" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7983d767", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:22:53.090665Z", + "iopub.status.busy": "2023-09-14T00:22:53.090326Z", + "iopub.status.idle": "2023-09-14T00:22:53.098892Z", + "shell.execute_reply": "2023-09-14T00:22:53.098217Z" + }, + "papermill": { + "duration": 0.015364, + "end_time": "2023-09-14T00:22:53.100628", + "exception": false, + "start_time": "2023-09-14T00:22:53.085264", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DEEPSPEED_STRAT: deepspeed_stage_1\n", + "ENABLE_WANDB: True\n", + "GPU_DEVICES: auto\n", + "DIR_NAME: L6-D2048-E1e-1-ctx4k\n", + "NOTEBOOK_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k\n", + "INFERENCE_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5\n", + "TRAINER_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5\n", + "PROJECT_DIR: /actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer\n" + ] + } + ], + "source": [ + "DEEPSPEED_STRAT=\"deepspeed_stage_1\"\n", + "GPU_DEVICES=\"auto\"\n", + "ENABLE_WANDB=True\n", + "\n", + "# Layer count and embed dim to start with\n", + "LAYER_COUNT=6\n", + "EMBED_DIM=2048\n", + "\n", + "EMBED_SCALE=0.1\n", + "EMBED_SCALE_LABEL=str(EMBED_SCALE).replace(\".\", \"_\")\n", + "\n", + "WANDB_PREFIX=f\"v5r3-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE}\"\n", + "FILENAME_PREFIX=f\"v5r3-L{LAYER_COUNT}-D{EMBED_DIM}-E{EMBED_SCALE_LABEL}\"\n", + "\n", + "print(\"DEEPSPEED_STRAT:\", DEEPSPEED_STRAT)\n", + "print(\"ENABLE_WANDB:\", ENABLE_WANDB)\n", + "print(\"GPU_DEVICES:\", GPU_DEVICES)\n", + "\n", + "if ENABLE_WANDB:\n", + " WANDB_MODE=\"online\"\n", + "else:\n", + " WANDB_MODE=\"disabled\"\n", + "\n", + "# Computing the notebook, and various paths\n", + "import os\n", + "NOTEBOOK_DIR=os.path.dirname(os.path.abspath(\"__file__\"))\n", + "CONFIG_DIR=os.path.abspath(os.path.join(NOTEBOOK_DIR, \"../\"))\n", + "PROJECT_DIR=os.path.abspath(os.path.join(CONFIG_DIR, \"../../../../\"))\n", + "TRAINER_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5/\"))\n", + "INFERENCE_DIR=os.path.abspath(os.path.join(PROJECT_DIR, \"./RWKV-v5/\"))\n", + "\n", + "# Get the notebook dir name\n", + "DIR_NAME=os.path.basename(NOTEBOOK_DIR)\n", + "\n", + "# Log names and dir\n", + "print(\"DIR_NAME:\", DIR_NAME)\n", + "print(\"NOTEBOOK_DIR:\", NOTEBOOK_DIR)\n", + "print(\"INFERENCE_DIR:\", INFERENCE_DIR)\n", + "print(\"TRAINER_DIR:\", TRAINER_DIR)\n", + "print(\"PROJECT_DIR:\", PROJECT_DIR)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "dd9d3898", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:22:53.110224Z", + "iopub.status.busy": "2023-09-14T00:22:53.109711Z", + "iopub.status.idle": "2023-09-14T00:23:17.707948Z", + "shell.execute_reply": "2023-09-14T00:23:17.707026Z" + }, + "papermill": { + "duration": 24.605816, + "end_time": "2023-09-14T00:23:17.710192", + "exception": false, + "start_time": "2023-09-14T00:22:53.104376", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "--2023-09-14 00:22:53-- https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/v5r3-L6-D2048-E0_1-mem-ctx-512.pth\r\n", + "Resolving huggingface.co (huggingface.co)... 18.154.227.69, 18.154.227.7, 18.154.227.67, ...\r\n", + "Connecting to huggingface.co (huggingface.co)|18.154.227.69|:443... connected.\r\n", + "HTTP request sent, awaiting response... 302 Found\r\n", + "Location: https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/92bee66e66bfcba8c592c785b63cb88f4e4889d78d7cdc49c33bd53bf0e3c31f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5r3-L6-D2048-E0_1-mem-ctx-512.pth%3B+filename%3D%22v5r3-L6-D2048-E0_1-mem-ctx-512.pth%22%3B&Expires=1694910173&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDkxMDE3M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzkyYmVlNjZlNjZiZmNiYThjNTkyYzc4NWI2M2NiODhmNGU0ODg5ZDc4ZDdjZGM0OWMzM2JkNTNiZjBlM2MzMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=GR1YnPvQoVKj9gwtKNiUTUL4%7E4P7pRpMbhWmjUzDMWgzkiYouaXIzAN9URl86Zv2bAbnLHd9g3j4Vzz0aNbRFg%7EEZMGj%7EkHKdT%7E6HjVQ-1d6uBBJHun8oQ%7ErvqtiKpAxNSj3DG9qRSuBcRi8aiqbirX1vu9VoUJ8bma73mERSKnLVCbNGe72BSAkZpgGmCMtJ1GfKjZ2qwEBm9ThwCf3dxQ9bf8Hlc54OUe5MnQTmEZ5xJVG6eUm191DjRXhRGlAWzLK2vb5QdaXbnQIR52e9dug1YN5h55zhHKIevguaq1DXuLqrqYOBrLiPpo3DE6UG0gXI88jQANFlenNQixV8A__&Key-Pair-Id=KVTP0A1DKRTAX [following]\r\n", + "--2023-09-14 00:22:53-- https://cdn-lfs.huggingface.co/repos/2e/f7/2ef78555202aa92abdbdf476ce3d0fd5a8b15f7245edf0b80d4d30572355f30d/92bee66e66bfcba8c592c785b63cb88f4e4889d78d7cdc49c33bd53bf0e3c31f?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27v5r3-L6-D2048-E0_1-mem-ctx-512.pth%3B+filename%3D%22v5r3-L6-D2048-E0_1-mem-ctx-512.pth%22%3B&Expires=1694910173&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY5NDkxMDE3M319LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9yZXBvcy8yZS9mNy8yZWY3ODU1NTIwMmFhOTJhYmRiZGY0NzZjZTNkMGZkNWE4YjE1ZjcyNDVlZGYwYjgwZDRkMzA1NzIzNTVmMzBkLzkyYmVlNjZlNjZiZmNiYThjNTkyYzc4NWI2M2NiODhmNGU0ODg5ZDc4ZDdjZGM0OWMzM2JkNTNiZjBlM2MzMWY%7EcmVzcG9uc2UtY29udGVudC1kaXNwb3NpdGlvbj0qIn1dfQ__&Signature=GR1YnPvQoVKj9gwtKNiUTUL4%7E4P7pRpMbhWmjUzDMWgzkiYouaXIzAN9URl86Zv2bAbnLHd9g3j4Vzz0aNbRFg%7EEZMGj%7EkHKdT%7E6HjVQ-1d6uBBJHun8oQ%7ErvqtiKpAxNSj3DG9qRSuBcRi8aiqbirX1vu9VoUJ8bma73mERSKnLVCbNGe72BSAkZpgGmCMtJ1GfKjZ2qwEBm9ThwCf3dxQ9bf8Hlc54OUe5MnQTmEZ5xJVG6eUm191DjRXhRGlAWzLK2vb5QdaXbnQIR52e9dug1YN5h55zhHKIevguaq1DXuLqrqYOBrLiPpo3DE6UG0gXI88jQANFlenNQixV8A__&Key-Pair-Id=KVTP0A1DKRTAX\r\n", + "Resolving cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)... 108.138.64.49, 108.138.64.111, 108.138.64.36, ...\r\n", + "Connecting to cdn-lfs.huggingface.co (cdn-lfs.huggingface.co)|108.138.64.49|:443... connected.\r\n", + "HTTP request sent, awaiting response... " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "200 OK\r\n", + "Length: 1066537217 (1017M) [binary/octet-stream]\r\n", + "Saving to: ‘v5r3-L6-D2048-E0_1-mem-ctx-512.pth’\r\n", + "\r\n", + "\r", + " v5r3-L6-D 0%[ ] 0 --.-KB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2 1%[ ] 14.74M 69.8MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D20 2%[ ] 24.19M 48.8MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D204 3%[ ] 30.52M 40.7MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048 3%[ ] 32.58M 28.7MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048- 3%[ ] 36.82M 27.2MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E 4%[ ] 45.26M 28.3MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0 5%[> ] 60.52M 31.9MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0_ 7%[> ] 74.59M 35.5MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0_1 7%[> ] 76.29M 30.8MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "v5r3-L6-D2048-E0_1- 8%[> ] 82.60M 29.0MB/s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "5r3-L6-D2048-E0_1-m 8%[> ] 91.03M 28.2MB/s eta 33s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "r3-L6-D2048-E0_1-me 9%[> ] 99.38M 29.0MB/s eta 33s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "3-L6-D2048-E0_1-mem 10%[=> ] 106.81M 29.0MB/s eta 33s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-L6-D2048-E0_1-mem- 12%[=> ] 122.07M 30.6MB/s eta 33s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "L6-D2048-E0_1-mem-c 13%[=> ] 137.33M 32.3MB/s eta 27s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "6-D2048-E0_1-mem-ct 15%[==> ] 152.59M 33.3MB/s eta 27s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-D2048-E0_1-mem-ctx 16%[==> ] 167.85M 32.6MB/s eta 27s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "D2048-E0_1-mem-ctx- 18%[==> ] 183.10M 34.6MB/s eta 27s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "2048-E0_1-mem-ctx-5 19%[==> ] 197.85M 38.9MB/s eta 22s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "048-E0_1-mem-ctx-51 20%[===> ] 213.11M 40.6MB/s eta 22s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "48-E0_1-mem-ctx-512 22%[===> ] 228.36M 40.9MB/s eta 22s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "8-E0_1-mem-ctx-512. 23%[===> ] 234.90M 41.6MB/s eta 22s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-E0_1-mem-ctx-512.p 25%[====> ] 257.58M 43.6MB/s eta 19s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "E0_1-mem-ctx-512.pt 25%[====> ] 259.40M 43.7MB/s eta 19s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "0_1-mem-ctx-512.pth 27%[====> ] 274.66M 49.4MB/s eta 19s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "_1-mem-ctx-512.pth 28%[====> ] 289.40M 51.8MB/s eta 19s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "1-mem-ctx-512.pth 29%[====> ] 304.66M 50.5MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-mem-ctx-512.pth 30%[=====> ] 307.24M 48.0MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "mem-ctx-512.pth 30%[=====> ] 311.49M 44.8MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "em-ctx-512.pth 31%[=====> ] 320.43M 42.2MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "m-ctx-512.pth 33%[=====> ] 335.69M 43.8MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-ctx-512.pth 34%[=====> ] 350.95M 44.2MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "ctx-512.pth 35%[======> ] 365.70M 42.5MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "tx-512.pth 37%[======> ] 376.60M 41.9MB/s eta 18s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "x-512.pth 37%[======> ] 381.59M 40.3MB/s eta 17s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-512.pth 39%[======> ] 396.73M 39.3MB/s eta 17s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "512.pth 40%[=======> ] 411.99M 41.6MB/s eta 17s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "12.pth 42%[=======> ] 427.25M 43.5MB/s eta 17s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "2.pth 43%[=======> ] 442.51M 43.0MB/s eta 17s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + ".pth 45%[========> ] 464.66M 44.5MB/s eta 13s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "pth 46%[========> ] 473.02M 45.6MB/s eta 13s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "th 47%[========> ] 486.45M 44.0MB/s eta 13s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "h 49%[========> ] 502.11M 44.8MB/s eta 13s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " 50%[=========> ] 510.22M 43.7MB/s eta 12s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v 51%[=========> ] 518.80M 45.3MB/s eta 12s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5 52%[=========> ] 534.05M 47.9MB/s eta 12s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r 53%[=========> ] 547.90M 48.5MB/s eta 12s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3 54%[=========> ] 555.93M 46.9MB/s eta 11s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3- 55%[==========> ] 564.58M 48.0MB/s eta 11s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L 56%[==========> ] 579.31M 49.2MB/s eta 11s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6 57%[==========> ] 579.96M 48.3MB/s eta 11s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6- 58%[==========> ] 595.09M 48.7MB/s eta 10s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D 60%[===========> ] 610.35M 47.1MB/s eta 10s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2 61%[===========> ] 625.09M 46.6MB/s eta 10s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D20 62%[===========> ] 634.25M 45.3MB/s eta 10s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D204 63%[===========> ] 640.87M 40.9MB/s eta 9s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048 64%[===========> ] 655.62M 43.0MB/s eta 9s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048- 65%[============> ] 666.06M 43.6MB/s eta 9s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E 65%[============> ] 669.55M 41.3MB/s eta 9s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0 66%[============> ] 671.38M 39.6MB/s eta 9s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0_ 67%[============> ] 686.13M 40.3MB/s eta 8s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " v5r3-L6-D2048-E0_1 68%[============> ] 701.39M 41.0MB/s eta 8s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "v5r3-L6-D2048-E0_1- 69%[============> ] 701.90M 39.7MB/s eta 8s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "5r3-L6-D2048-E0_1-m 70%[=============> ] 717.16M 39.8MB/s eta 8s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "r3-L6-D2048-E0_1-me 72%[=============> ] 732.42M 39.2MB/s eta 7s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "3-L6-D2048-E0_1-mem 73%[=============> ] 747.69M 41.0MB/s eta 7s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-L6-D2048-E0_1-mem- 75%[==============> ] 762.94M 42.2MB/s eta 7s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "L6-D2048-E0_1-mem-c 76%[==============> ] 777.68M 43.3MB/s eta 7s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "6-D2048-E0_1-mem-ct 76%[==============> ] 780.57M 40.6MB/s eta 7s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-D2048-E0_1-mem-ctx 77%[==============> ] 792.94M 42.4MB/s eta 5s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "D2048-E0_1-mem-ctx- 78%[==============> ] 798.64M 41.9MB/s eta 5s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "2048-E0_1-mem-ctx-5 79%[==============> ] 808.20M 42.8MB/s eta 5s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "048-E0_1-mem-ctx-51 80%[===============> ] 823.46M 42.0MB/s eta 5s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "48-E0_1-mem-ctx-512 82%[===============> ] 838.71M 42.8MB/s eta 4s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "8-E0_1-mem-ctx-512. 82%[===============> ] 839.23M 42.0MB/s eta 4s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-E0_1-mem-ctx-512.p 85%[================> ] 867.93M 45.9MB/s eta 4s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "E0_1-mem-ctx-512.pt 87%[================> ] 885.01M 48.1MB/s eta 4s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "0_1-mem-ctx-512.pth 87%[================> ] 893.44M 47.0MB/s eta 3s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "_1-mem-ctx-512.pth 88%[================> ] 899.75M 48.1MB/s eta 3s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "1-mem-ctx-512.pth 89%[================> ] 906.74M 46.0MB/s eta 3s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-mem-ctx-512.pth 90%[=================> ] 917.87M 48.3MB/s eta 3s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "mem-ctx-512.pth 91%[=================> ] 930.78M 46.4MB/s eta 3s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "em-ctx-512.pth 92%[=================> ] 944.21M 47.3MB/s eta 2s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "m-ctx-512.pth 93%[=================> ] 955.92M 46.9MB/s eta 2s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-ctx-512.pth 94%[=================> ] 961.30M 44.8MB/s eta 2s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "ctx-512.pth 96%[==================> ] 976.55M 48.0MB/s eta 2s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "tx-512.pth 96%[==================> ] 986.36M 47.3MB/s eta 2s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "x-512.pth 97%[==================> ] 993.01M 48.4MB/s eta 1s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "-512.pth 99%[==================> ] 1007M 44.3MB/s eta 1s " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "v5r3-L6-D2048-E0_1- 100%[===================>] 1017M 45.5MB/s in 24s \r\n", + "\r\n", + "2023-09-14 00:23:17 (42.7 MB/s) - ‘v5r3-L6-D2048-E0_1-mem-ctx-512.pth’ saved [1066537217/1066537217]\r\n", + "\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 1018M\r\n", + "drwxr-xr-x 2 root root 4.0K Sep 14 00:22 .\r\n", + "drwxr-xr-x 20 root root 4.0K Sep 14 00:22 ..\r\n", + "-rw-r--r-- 1 root root 1018M Sep 13 20:28 v5r3-L6-D2048-E0_1-mem-ctx-512.pth\r\n" + ] + } + ], + "source": [ + "# Download the model directly (stop gap till HF sync issues is resolved)\n", + "!cd \"{TRAINER_DIR}\" && cd \"../model/\" && \\\n", + " wget -nc \"https://huggingface.co/rwkv-x-dev/rwkv-x-playground/resolve/main/experiment/rwkv-x-exp/v5-r3-memory/{DIR_NAME}/{FILENAME_PREFIX}-mem-ctx-512.pth\"\n", + "\n", + "!cd \"{TRAINER_DIR}\" && cd \"../model/\" && \\\n", + " ls -alh ." + ] + }, + { + "cell_type": "markdown", + "id": "c2d7023a", + "metadata": { + "papermill": { + "duration": 0.008067, + "end_time": "2023-09-14T00:23:17.726510", + "exception": false, + "start_time": "2023-09-14T00:23:17.718443", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Tune 3 : Ramping up the ctx size (8192), memory training\n", + "\n", + "- Tune 3: Large ctx size (8192), Scaling up!" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f827262e", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:23:17.745636Z", + "iopub.status.busy": "2023-09-14T00:23:17.744822Z", + "iopub.status.idle": "2023-09-14T00:23:38.055622Z", + "shell.execute_reply": "2023-09-14T00:23:38.054557Z" + }, + "papermill": { + "duration": 20.338726, + "end_time": "2023-09-14T00:23:38.073537", + "exception": false, + "start_time": "2023-09-14T00:23:17.734811", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "## Generating word reptition dataset ##\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 40 max words, 100 samples - at ../dataset/gen-word-40-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 50 max words, 100 samples - at ../dataset/gen-word-50-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5 max words, 100 samples - at ../dataset/gen-word-5-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 20 max words, 100 samples - at ../dataset/gen-word-20-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 60 max words, 100 samples - at ../dataset/gen-word-60-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 10 max words, 100 samples - at ../dataset/gen-word-10-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 90 max words, 100 samples - at ../dataset/gen-word-90-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 518 samples (10 token repeat) - 50 max words - at ../dataset/shuffle-word-50-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 15 max words, 100 samples - at ../dataset/gen-word-15-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 25 max words, 100 samples - at ../dataset/gen-word-25-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 35 max words, 100 samples - at ../dataset/gen-word-35-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 45 max words, 100 samples - at ../dataset/gen-word-45-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 749 samples (10 token repeat) - 35 max words - at ../dataset/shuffle-word-35-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 30 max words, 100 samples - at ../dataset/gen-word-30-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 85 max words, 100 samples - at ../dataset/gen-word-85-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 478 samples (10 token repeat) - 55 max words - at ../dataset/shuffle-word-55-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 651 samples (10 token repeat) - 40 max words - at ../dataset/shuffle-word-40-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5100 max words - at ../dataset/shuffle-word-5100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 1057 samples (10 token repeat) - 25 max words - at ../dataset/shuffle-word-25-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 185 samples (20 token repeat) - 300 max words - at ../dataset/shuffle-word-300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 100 max words, 100 samples - at ../dataset/gen-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4300 max words - at ../dataset/shuffle-word-4300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 590 samples (10 token repeat) - 45 max words - at ../dataset/shuffle-word-45-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 308 samples (10 token repeat) - 85 max words - at ../dataset/shuffle-word-85-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3100 max words - at ../dataset/shuffle-word-3100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 1304 samples (10 token repeat) - 20 max words - at ../dataset/shuffle-word-20-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 60 samples (20 token repeat) - 1200 max words - at ../dataset/shuffle-word-1200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 531 samples (20 token repeat) - 100 max words - at ../dataset/shuffle-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 95 max words, 100 samples - at ../dataset/gen-word-95-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 882 samples (10 token repeat) - 30 max words - at ../dataset/shuffle-word-30-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6800 max words - at ../dataset/shuffle-word-6800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 2300 max words - at ../dataset/shuffle-word-2300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6500 max words - at ../dataset/shuffle-word-6500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 379 samples (10 token repeat) - 70 max words - at ../dataset/shuffle-word-70-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5900 max words - at ../dataset/shuffle-word-5900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 414 samples (10 token repeat) - 65 max words - at ../dataset/shuffle-word-65-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 1791 samples (10 token repeat) - 15 max words - at ../dataset/shuffle-word-15-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 65 max words, 100 samples - at ../dataset/gen-word-65-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7900 max words - at ../dataset/shuffle-word-7900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 80 max words, 100 samples - at ../dataset/gen-word-80-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 2400 max words - at ../dataset/shuffle-word-2400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 354 samples (10 token repeat) - 75 max words - at ../dataset/shuffle-word-75-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 332 samples (10 token repeat) - 80 max words - at ../dataset/shuffle-word-80-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 70 max words, 100 samples - at ../dataset/gen-word-70-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 75 max words, 100 samples - at ../dataset/gen-word-75-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 304 samples (10 token repeat) - 90 max words - at ../dataset/shuffle-word-90-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 2100 max words - at ../dataset/shuffle-word-2100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 100 samples (20 token repeat) - 600 max words - at ../dataset/shuffle-word-600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 55 max words, 100 samples - at ../dataset/gen-word-55-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3200 max words - at ../dataset/shuffle-word-3200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7000 max words - at ../dataset/shuffle-word-7000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5400 max words - at ../dataset/shuffle-word-5400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5500 max words - at ../dataset/shuffle-word-5500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6400 max words - at ../dataset/shuffle-word-6400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4700 max words - at ../dataset/shuffle-word-4700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4900 max words - at ../dataset/shuffle-word-4900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 269 samples (20 token repeat) - 200 max words - at ../dataset/shuffle-word-200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 21 samples (20 token repeat) - 2700 max words - at ../dataset/shuffle-word-2700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 41 samples (20 token repeat) - 1400 max words - at ../dataset/shuffle-word-1400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6100 max words - at ../dataset/shuffle-word-6100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6000 max words - at ../dataset/shuffle-word-6000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7500 max words - at ../dataset/shuffle-word-7500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6700 max words - at ../dataset/shuffle-word-6700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7400 max words - at ../dataset/shuffle-word-7400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 31 samples (20 token repeat) - 2600 max words - at ../dataset/shuffle-word-2600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3700 max words - at ../dataset/shuffle-word-3700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5600 max words - at ../dataset/shuffle-word-5600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 2592 samples (10 token repeat) - 10 max words - at ../dataset/shuffle-word-10-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 21 samples (20 token repeat) - 2800 max words - at ../dataset/shuffle-word-2800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3500 max words - at ../dataset/shuffle-word-3500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 283 samples (10 token repeat) - 95 max words - at ../dataset/shuffle-word-95-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 60 samples (20 token repeat) - 1000 max words - at ../dataset/shuffle-word-1000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4600 max words - at ../dataset/shuffle-word-4600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3400 max words - at ../dataset/shuffle-word-3400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 2000 max words - at ../dataset/shuffle-word-2000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6300 max words - at ../dataset/shuffle-word-6300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4000 max words - at ../dataset/shuffle-word-4000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4500 max words - at ../dataset/shuffle-word-4500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 60 samples (20 token repeat) - 1100 max words - at ../dataset/shuffle-word-1100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7100 max words - at ../dataset/shuffle-word-7100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6900 max words - at ../dataset/shuffle-word-6900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3800 max words - at ../dataset/shuffle-word-3800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 1900 max words - at ../dataset/shuffle-word-1900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 5561 samples (10 token repeat) - 5 max words - at ../dataset/shuffle-word-5-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3600 max words - at ../dataset/shuffle-word-3600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 2200 max words - at ../dataset/shuffle-word-2200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3300 max words - at ../dataset/shuffle-word-3300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5700 max words - at ../dataset/shuffle-word-5700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5800 max words - at ../dataset/shuffle-word-5800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6200 max words - at ../dataset/shuffle-word-6200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 82 samples (20 token repeat) - 700 max words - at ../dataset/shuffle-word-700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5000 max words - at ../dataset/shuffle-word-5000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 438 samples (10 token repeat) - 60 max words - at ../dataset/shuffle-word-60-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5300 max words - at ../dataset/shuffle-word-5300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 35 samples (20 token repeat) - 2500 max words - at ../dataset/shuffle-word-2500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 119 samples (20 token repeat) - 500 max words - at ../dataset/shuffle-word-500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 8000 max words - at ../dataset/shuffle-word-8000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4100 max words - at ../dataset/shuffle-word-4100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 1600 max words - at ../dataset/shuffle-word-1600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7600 max words - at ../dataset/shuffle-word-7600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4400 max words - at ../dataset/shuffle-word-4400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 5200 max words - at ../dataset/shuffle-word-5200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 1500 max words - at ../dataset/shuffle-word-1500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 1700 max words - at ../dataset/shuffle-word-1700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3000 max words - at ../dataset/shuffle-word-3000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 40 samples (20 token repeat) - 1800 max words - at ../dataset/shuffle-word-1800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 268 samples (10 token repeat) - 100 max words - at ../dataset/shuffle-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 6600 max words - at ../dataset/shuffle-word-6600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 140 samples (20 token repeat) - 400 max words - at ../dataset/shuffle-word-400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4200 max words - at ../dataset/shuffle-word-4200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 4800 max words - at ../dataset/shuffle-word-4800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 80 samples (20 token repeat) - 800 max words - at ../dataset/shuffle-word-800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 52 samples (20 token repeat) - 1300 max words - at ../dataset/shuffle-word-1300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7300 max words - at ../dataset/shuffle-word-7300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 62 samples (20 token repeat) - 900 max words - at ../dataset/shuffle-word-900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7700 max words - at ../dataset/shuffle-word-7700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 21 samples (20 token repeat) - 2900 max words - at ../dataset/shuffle-word-2900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7200 max words - at ../dataset/shuffle-word-7200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 7800 max words - at ../dataset/shuffle-word-7800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated a single JSONL file with 20 samples (20 token repeat) - 3900 max words - at ../dataset/shuffle-word-3900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 100 max words, 2000 samples - at ../dataset/gen-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 200 max words, 2000 samples - at ../dataset/gen-word-200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 300 max words, 2000 samples - at ../dataset/gen-word-300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 700 max words, 2000 samples - at ../dataset/gen-word-700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 400 max words, 2000 samples - at ../dataset/gen-word-400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 800 max words, 2000 samples - at ../dataset/gen-word-800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 500 max words, 2000 samples - at ../dataset/gen-word-500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1400 max words, 2000 samples - at ../dataset/gen-word-1400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 900 max words, 2000 samples - at ../dataset/gen-word-900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 600 max words, 2000 samples - at ../dataset/gen-word-600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1000 max words, 2000 samples - at ../dataset/gen-word-1000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1100 max words, 2000 samples - at ../dataset/gen-word-1100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1300 max words, 2000 samples - at ../dataset/gen-word-1300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1200 max words, 2000 samples - at ../dataset/gen-word-1200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3100 max words, 2000 samples - at ../dataset/gen-word-3100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1600 max words, 2000 samples - at ../dataset/gen-word-1600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1500 max words, 2000 samples - at ../dataset/gen-word-1500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2900 max words, 2000 samples - at ../dataset/gen-word-2900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2200 max words, 2000 samples - at ../dataset/gen-word-2200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2500 max words, 2000 samples - at ../dataset/gen-word-2500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3000 max words, 2000 samples - at ../dataset/gen-word-3000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1700 max words, 2000 samples - at ../dataset/gen-word-1700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2000 max words, 2000 samples - at ../dataset/gen-word-2000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1800 max words, 2000 samples - at ../dataset/gen-word-1800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 1900 max words, 2000 samples - at ../dataset/gen-word-1900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2100 max words, 2000 samples - at ../dataset/gen-word-2100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3300 max words, 2000 samples - at ../dataset/gen-word-3300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5400 max words, 2000 samples - at ../dataset/gen-word-5400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2600 max words, 2000 samples - at ../dataset/gen-word-2600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2300 max words, 2000 samples - at ../dataset/gen-word-2300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3200 max words, 2000 samples - at ../dataset/gen-word-3200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4200 max words, 2000 samples - at ../dataset/gen-word-4200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2700 max words, 2000 samples - at ../dataset/gen-word-2700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4000 max words, 2000 samples - at ../dataset/gen-word-4000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2400 max words, 2000 samples - at ../dataset/gen-word-2400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3400 max words, 2000 samples - at ../dataset/gen-word-3400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5100 max words, 2000 samples - at ../dataset/gen-word-5100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4400 max words, 2000 samples - at ../dataset/gen-word-4400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5000 max words, 2000 samples - at ../dataset/gen-word-5000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3900 max words, 2000 samples - at ../dataset/gen-word-3900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 2800 max words, 2000 samples - at ../dataset/gen-word-2800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7400 max words, 2000 samples - at ../dataset/gen-word-7400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4100 max words, 2000 samples - at ../dataset/gen-word-4100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4800 max words, 2000 samples - at ../dataset/gen-word-4800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4700 max words, 2000 samples - at ../dataset/gen-word-4700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3600 max words, 2000 samples - at ../dataset/gen-word-3600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3800 max words, 2000 samples - at ../dataset/gen-word-3800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6200 max words, 2000 samples - at ../dataset/gen-word-6200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3700 max words, 2000 samples - at ../dataset/gen-word-3700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 3500 max words, 2000 samples - at ../dataset/gen-word-3500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6400 max words, 2000 samples - at ../dataset/gen-word-6400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7700 max words, 2000 samples - at ../dataset/gen-word-7700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5500 max words, 2000 samples - at ../dataset/gen-word-5500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4500 max words, 2000 samples - at ../dataset/gen-word-4500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4600 max words, 2000 samples - at ../dataset/gen-word-4600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4300 max words, 2000 samples - at ../dataset/gen-word-4300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6000 max words, 2000 samples - at ../dataset/gen-word-6000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5300 max words, 2000 samples - at ../dataset/gen-word-5300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5900 max words, 2000 samples - at ../dataset/gen-word-5900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 4900 max words, 2000 samples - at ../dataset/gen-word-4900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7600 max words, 2000 samples - at ../dataset/gen-word-7600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6700 max words, 2000 samples - at ../dataset/gen-word-6700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6600 max words, 2000 samples - at ../dataset/gen-word-6600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6800 max words, 2000 samples - at ../dataset/gen-word-6800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5200 max words, 2000 samples - at ../dataset/gen-word-5200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6900 max words, 2000 samples - at ../dataset/gen-word-6900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6100 max words, 2000 samples - at ../dataset/gen-word-6100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7100 max words, 2000 samples - at ../dataset/gen-word-7100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5700 max words, 2000 samples - at ../dataset/gen-word-5700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6500 max words, 2000 samples - at ../dataset/gen-word-6500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5600 max words, 2000 samples - at ../dataset/gen-word-5600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 5800 max words, 2000 samples - at ../dataset/gen-word-5800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 6300 max words, 2000 samples - at ../dataset/gen-word-6300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7500 max words, 2000 samples - at ../dataset/gen-word-7500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7800 max words, 2000 samples - at ../dataset/gen-word-7800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7000 max words, 2000 samples - at ../dataset/gen-word-7000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7900 max words, 2000 samples - at ../dataset/gen-word-7900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7200 max words, 2000 samples - at ../dataset/gen-word-7200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 7300 max words, 2000 samples - at ../dataset/gen-word-7300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated JSONL file with - 8000 max words, 2000 samples - at ../dataset/gen-word-8000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "## Done ##\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 6.1G\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 20K Sep 14 00:23 gen-word-10-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 2.1M Sep 14 00:23 gen-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 20M Sep 14 00:23 gen-word-1000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 22M Sep 14 00:23 gen-word-1100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 23M Sep 14 00:23 gen-word-1200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 25M Sep 14 00:23 gen-word-1300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 27M Sep 14 00:23 gen-word-1400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 24K Sep 14 00:23 gen-word-15-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 29M Sep 14 00:23 gen-word-1500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 31M Sep 14 00:23 gen-word-1600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 33M Sep 14 00:23 gen-word-1700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 35M Sep 14 00:23 gen-word-1800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 37M Sep 14 00:23 gen-word-1900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 30K Sep 14 00:23 gen-word-20-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 4.0M Sep 14 00:23 gen-word-200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 39M Sep 14 00:23 gen-word-2000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 41M Sep 14 00:23 gen-word-2100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 42M Sep 14 00:23 gen-word-2200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 44M Sep 14 00:23 gen-word-2300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 46M Sep 14 00:23 gen-word-2400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 35K Sep 14 00:23 gen-word-25-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 48M Sep 14 00:23 gen-word-2500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 50M Sep 14 00:23 gen-word-2600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 52M Sep 14 00:23 gen-word-2700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 54M Sep 14 00:23 gen-word-2800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 56M Sep 14 00:23 gen-word-2900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 39K Sep 14 00:23 gen-word-30-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 5.9M Sep 14 00:23 gen-word-300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 58M Sep 14 00:23 gen-word-3000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 60M Sep 14 00:23 gen-word-3100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 61M Sep 14 00:23 gen-word-3200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 63M Sep 14 00:23 gen-word-3300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 65M Sep 14 00:23 gen-word-3400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 44K Sep 14 00:23 gen-word-35-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 67M Sep 14 00:23 gen-word-3500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 69M Sep 14 00:23 gen-word-3600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 71M Sep 14 00:23 gen-word-3700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 73M Sep 14 00:23 gen-word-3800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 75M Sep 14 00:23 gen-word-3900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 48K Sep 14 00:23 gen-word-40-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 7.8M Sep 14 00:23 gen-word-400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 77M Sep 14 00:23 gen-word-4000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 79M Sep 14 00:23 gen-word-4100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 80M Sep 14 00:23 gen-word-4200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 82M Sep 14 00:23 gen-word-4300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 84M Sep 14 00:23 gen-word-4400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 54K Sep 14 00:23 gen-word-45-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 86M Sep 14 00:23 gen-word-4500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 88M Sep 14 00:23 gen-word-4600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 90M Sep 14 00:23 gen-word-4700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 92M Sep 14 00:23 gen-word-4800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 94M Sep 14 00:23 gen-word-4900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 15K Sep 14 00:23 gen-word-5-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 59K Sep 14 00:23 gen-word-50-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 9.7M Sep 14 00:23 gen-word-500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 96M Sep 14 00:23 gen-word-5000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 98M Sep 14 00:23 gen-word-5100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 99M Sep 14 00:23 gen-word-5200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 101M Sep 14 00:23 gen-word-5300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 103M Sep 14 00:23 gen-word-5400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 63K Sep 14 00:23 gen-word-55-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 105M Sep 14 00:23 gen-word-5500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 107M Sep 14 00:23 gen-word-5600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 109M Sep 14 00:23 gen-word-5700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 111M Sep 14 00:23 gen-word-5800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 113M Sep 14 00:23 gen-word-5900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 70K Sep 14 00:23 gen-word-60-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 12M Sep 14 00:23 gen-word-600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 115M Sep 14 00:23 gen-word-6000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 117M Sep 14 00:23 gen-word-6100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 119M Sep 14 00:23 gen-word-6200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 121M Sep 14 00:23 gen-word-6300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 122M Sep 14 00:23 gen-word-6400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 72K Sep 14 00:23 gen-word-65-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 124M Sep 14 00:23 gen-word-6500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 126M Sep 14 00:23 gen-word-6600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 128M Sep 14 00:23 gen-word-6700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 130M Sep 14 00:23 gen-word-6800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 132M Sep 14 00:23 gen-word-6900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 74K Sep 14 00:23 gen-word-70-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 14M Sep 14 00:23 gen-word-700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 134M Sep 14 00:23 gen-word-7000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 135M Sep 14 00:23 gen-word-7100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 138M Sep 14 00:23 gen-word-7200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 139M Sep 14 00:23 gen-word-7300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 141M Sep 14 00:23 gen-word-7400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 81K Sep 14 00:23 gen-word-75-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 143M Sep 14 00:23 gen-word-7500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 145M Sep 14 00:23 gen-word-7600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 147M Sep 14 00:23 gen-word-7700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 149M Sep 14 00:23 gen-word-7800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 151M Sep 14 00:23 gen-word-7900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 88K Sep 14 00:23 gen-word-80-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 16M Sep 14 00:23 gen-word-800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 153M Sep 14 00:23 gen-word-8000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 90K Sep 14 00:23 gen-word-85-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 95K Sep 14 00:23 gen-word-90-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 18M Sep 14 00:23 gen-word-900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 101K Sep 14 00:23 gen-word-95-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 507K Sep 14 00:23 shuffle-word-10-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 288K Sep 14 00:23 shuffle-word-100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 524K Sep 14 00:23 shuffle-word-1000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 521K Sep 14 00:23 shuffle-word-1100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 521K Sep 14 00:23 shuffle-word-1200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 525K Sep 14 00:23 shuffle-word-1300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 522K Sep 14 00:23 shuffle-word-1400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 433K Sep 14 00:23 shuffle-word-15-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 521K Sep 14 00:23 shuffle-word-1500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 518K Sep 14 00:23 shuffle-word-1600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 522K Sep 14 00:23 shuffle-word-1700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 521K Sep 14 00:23 shuffle-word-1800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 522K Sep 14 00:23 shuffle-word-1900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 386K Sep 14 00:23 shuffle-word-20-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 540K Sep 14 00:23 shuffle-word-200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 518K Sep 14 00:23 shuffle-word-2000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 522K Sep 14 00:23 shuffle-word-2100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 519K Sep 14 00:23 shuffle-word-2200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 519K Sep 14 00:23 shuffle-word-2300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 520K Sep 14 00:23 shuffle-word-2400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 363K Sep 14 00:23 shuffle-word-25-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 519K Sep 14 00:23 shuffle-word-2500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 516K Sep 14 00:23 shuffle-word-2600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-2700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 510K Sep 14 00:23 shuffle-word-2800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 510K Sep 14 00:23 shuffle-word-2900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 336K Sep 14 00:23 shuffle-word-30-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 535K Sep 14 00:23 shuffle-word-300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-3200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 331K Sep 14 00:23 shuffle-word-35-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-3700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-3900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 321K Sep 14 00:23 shuffle-word-40-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 529K Sep 14 00:23 shuffle-word-400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-4100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 311K Sep 14 00:23 shuffle-word-45-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-4900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 813K Sep 14 00:23 shuffle-word-5-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 308K Sep 14 00:23 shuffle-word-50-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 529K Sep 14 00:23 shuffle-word-500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 309K Sep 14 00:23 shuffle-word-55-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-5900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 300K Sep 14 00:23 shuffle-word-60-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 527K Sep 14 00:23 shuffle-word-600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-6100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 300K Sep 14 00:23 shuffle-word-65-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-6600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-6700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-6800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 508K Sep 14 00:23 shuffle-word-6900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 295K Sep 14 00:23 shuffle-word-70-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 527K Sep 14 00:23 shuffle-word-700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7100-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7200-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7300-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7400-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 288K Sep 14 00:23 shuffle-word-75-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7500-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7600-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7700-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-7900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 290K Sep 14 00:23 shuffle-word-80-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 523K Sep 14 00:23 shuffle-word-800-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 509K Sep 14 00:23 shuffle-word-8000-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 284K Sep 14 00:23 shuffle-word-85-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 290K Sep 14 00:23 shuffle-word-90-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 525K Sep 14 00:23 shuffle-word-900-count.jsonl\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-rw-r--r-- 1 root root 288K Sep 14 00:23 shuffle-word-95-count.jsonl\n" + ] + } + ], + "source": [ + "%%script bash\n", + "\n", + "########################################\n", + "# Generate the required jsonl dataset\n", + "########################################\n", + "\n", + "# Go to config dir\n", + "cd \"../\"\n", + "\n", + "# Reset the dataset dir\n", + "mkdir -p ../dataset\n", + "rm -rf ../dataset/*.jsonl\n", + "\n", + "# Generate the various datasets\n", + "echo \"## Generating word reptition dataset ##\"\n", + "\n", + "#\n", + "# We reduce the training set for < 50 words - and shift the focus upwards\n", + "# (aka 50-100 token * 2 : ~100 - 250 token ctx len)\n", + "#\n", + "for i in {5..100..5} \n", + "do\n", + " python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/gen-word-$i-count.jsonl $i 100 & \n", + " python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-$i-count.jsonl $i 10 & \n", + "done\n", + "\n", + "#\n", + "# Ramping up the 100+ - 4200 words dataset\n", + "# \n", + "for i in {100..8000..100} \n", + "do\n", + " python3 ../memory_script/gen_limited_prompt_completion_jsonl.py ../dataset/gen-word-$i-count.jsonl $i 2000 & \n", + " python3 ../memory_script/shuffle_limited_prompt_completion_jsonl.py ../dataset/shuffle-word-$i-count.jsonl $i 20 & \n", + "done\n", + "\n", + "wait\n", + "echo \"## Done ##\"\n", + "\n", + "ls -lh ../dataset/" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "dc750aad", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:23:38.144140Z", + "iopub.status.busy": "2023-09-14T00:23:38.143744Z", + "iopub.status.idle": "2023-09-14T00:24:09.381956Z", + "shell.execute_reply": "2023-09-14T00:24:09.380782Z" + }, + "papermill": { + "duration": 31.277099, + "end_time": "2023-09-14T00:24:09.384289", + "exception": false, + "start_time": "2023-09-14T00:23:38.107190", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-09-14 00:23:43,047] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[RWKV.model] Running RWKV model using 'torch-jit' with torch '2.0.1+cu118'\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/lightning/pytorch/cli.py:484: UserWarning: LightningCLI's args parameter is intended to run from within Python like if it were from the command line. To prevent mistakes it is not recommended to provide both args and command line arguments, got: sys.argv[1:]=['fit', '-c', '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/config-mem-template.yaml', '--trainer.logger.init_args.name=v5r3-L6-D2048-E0.1 - Mem-Tune ctx-8k (train-ctx=4k, deepspeed_stage_1)', '--trainer.strategy=deepspeed_stage_1', '--trainer.devices=auto', '--trainer.callbacks.init_args.dirpath=../checkpoint/v5r3-L6-D2048-E0_1-mem-ctx-8k/', '--model.lr_init=4e-4', '--model.lr_final=2e-4', '--data.max_token_size=8192', '--data.sort_by_length=True', '--model.ctx_len=4096', '--model.bptt_learning_range=2', '--model.load_model=../model/v5r3-L6-D2048-E0_1-mem-ctx-512.pth'], args=['fit', '-c', '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/config-mem-template.yaml', '--trainer.logger.init_args.name=v5r3-L6-D2048-E0.1 - Mem-Tune ctx-8k (train-ctx=4k, deepspeed_stage_1)', '--trainer.strategy=deepspeed_stage_1', '--trainer.devices=auto', '--trainer.callbacks.init_args.dirpath=../checkpoint/v5r3-L6-D2048-E0_1-mem-ctx-8k/', '--model.lr_init=4e-4', '--model.lr_final=2e-4', '--data.max_token_size=8192', '--data.sort_by_length=True', '--model.ctx_len=4096', '--model.bptt_learning_range=2', '--model.load_model=../model/v5r3-L6-D2048-E0_1-mem-ctx-512.pth'].\r\n", + " rank_zero_warn(\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/lightning/fabric/utilities/seed.py:39: UserWarning: No seed found, seed set to 2409510691\r\n", + " rank_zero_warn(f\"No seed found, seed set to {seed}\")\r\n", + "Global seed set to 2409510691\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mpicocreator\u001b[0m (\u001b[33mrwkv-x-dev\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.15.10\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m./wandb/run-20230914_002346-rjj8azcb\u001b[0m\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mv5r3-L6-D2048-E0.1 - Mem-Tune ctx-8k (train-ctx=4k, deepspeed_stage_1)\u001b[0m\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments\u001b[0m\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments/runs/rjj8azcb\u001b[0m\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/lightning/fabric/connector.py:554: UserWarning: bf16 is supported for historical reasons but its usage is discouraged. Please set your precision to bf16-mixed instead!\r\n", + " rank_zero_warn(\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPU available: True (cuda), used: True\r\n", + "TPU available: False, using: 0 TPU cores\r\n", + "IPU available: False, using: 0 IPUs\r\n", + "HPU available: False, using: 0 HPUs\r\n", + "\r\n", + "\r\n", + "[RWKV.Trainer] Applying 'target_batch_size' with the following:\r\n", + " - target_batch_size: 256\r\n", + " - num_nodes: 1\r\n", + " - num_devices: 1\r\n", + " - accumulate_grad_batches: 256\r\n", + " - effective_batch_size: 256\r\n", + "\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Resolving data files: 0%| | 0/198 [00:00: JSON parse error: The document is empty.\r\n", + "\r", + "Generating train split: 100 examples [00:01, 59.81 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 4192 examples [00:01, 3264.19 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 7274 examples [00:01, 5941.16 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 12298 examples [00:02, 11319.63 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 16646 examples [00:02, 15857.43 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 24754 examples [00:02, 25730.84 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 29784 examples [00:02, 29814.87 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 51040 examples [00:02, 67851.73 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 60766 examples [00:02, 69517.43 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 69699 examples [00:02, 61870.64 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 77449 examples [00:02, 55006.72 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 84355 examples [00:03, 52194.78 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 90582 examples [00:03, 52818.03 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 96557 examples [00:03, 49398.87 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 101972 examples [00:03, 44994.14 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 106761 examples [00:03, 42524.57 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 111314 examples [00:03, 40451.50 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 115495 examples [00:03, 38042.11 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 121217 examples [00:04, 42265.52 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 125781 examples [00:04, 38604.07 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 131059 examples [00:04, 41930.09 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 135527 examples [00:04, 37367.60 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 140337 examples [00:04, 39261.42 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 144530 examples [00:04, 38141.40 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 148531 examples [00:04, 35123.00 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 152149 examples [00:04, 32635.68 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 155588 examples [00:05, 28239.31 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 158703 examples [00:05, 25877.65 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 161402 examples [00:05, 25746.47 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 164146 examples [00:05, 23432.26 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 166599 examples [00:05, 22161.96 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 169044 examples [00:05, 18107.60 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 171109 examples [00:06, 13602.32 examples/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Generating train split: 172012 examples [00:06, 26818.32 examples/s]\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "multiprocess.pool.RemoteTraceback: \r\n", + "\"\"\"\r\n", + "Traceback (most recent call last):\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/packaged_modules/json/json.py\", line 144, in _generate_tables\r\n", + " dataset = json.load(f)\r\n", + " File \"/usr/lib/python3.10/json/__init__.py\", line 293, in load\r\n", + " return loads(fp.read(),\r\n", + " File \"/usr/lib/python3.10/json/__init__.py\", line 346, in loads\r\n", + " return _default_decoder.decode(s)\r\n", + " File \"/usr/lib/python3.10/json/decoder.py\", line 340, in decode\r\n", + " raise JSONDecodeError(\"Extra data\", s, end)\r\n", + "json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 909)\r\n", + "\r\n", + "During handling of the above exception, another exception occurred:\r\n", + "\r\n", + "Traceback (most recent call last):\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/builder.py\", line 1925, in _prepare_split_single\r\n", + " for _, table in generator:\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/packaged_modules/json/json.py\", line 147, in _generate_tables\r\n", + " raise e\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/packaged_modules/json/json.py\", line 121, in _generate_tables\r\n", + " pa_table = paj.read_json(\r\n", + " File \"pyarrow/_json.pyx\", line 258, in pyarrow._json.read_json\r\n", + " File \"pyarrow/error.pxi\", line 144, in pyarrow.lib.pyarrow_internal_check_status\r\n", + " File \"pyarrow/error.pxi\", line 100, in pyarrow.lib.check_status\r\n", + "pyarrow.lib.ArrowInvalid: JSON parse error: The document is empty.\r\n", + "\r\n", + "The above exception was the direct cause of the following exception:\r\n", + "\r\n", + "Traceback (most recent call last):\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/multiprocess/pool.py\", line 125, in worker\r\n", + " result = (True, func(*args, **kwds))\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/utils/py_utils.py\", line 1347, in _write_generator_to_queue\r\n", + " for i, result in enumerate(func(**kwargs)):\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/builder.py\", line 1958, in _prepare_split_single\r\n", + " raise DatasetGenerationError(\"An error occurred while generating the dataset\") from e\r\n", + "datasets.builder.DatasetGenerationError: An error occurred while generating the dataset\r\n", + "\"\"\"\r\n", + "\r\n", + "The above exception was the direct cause of the following exception:\r\n", + "\r\n", + "Traceback (most recent call last):\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/lightning_trainer.py\", line 258, in \r\n", + " cli_main()\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/lightning_trainer.py\", line 233, in cli_main\r\n", + " LightningCLI(\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/cli.py\", line 353, in __init__\r\n", + " self._run_subcommand(self.subcommand)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/cli.py\", line 642, in _run_subcommand\r\n", + " fn(**fn_kwargs)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/trainer.py\", line 529, in fit\r\n", + " call._call_and_handle_interrupt(\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/call.py\", line 41, in _call_and_handle_interrupt\r\n", + " return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/strategies/launchers/subprocess_script.py\", line 91, in launch\r\n", + " return function(*args, **kwargs)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/trainer.py\", line 568, in _fit_impl\r\n", + " self._run(model, ckpt_path=ckpt_path)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/trainer.py\", line 925, in _run\r\n", + " self._data_connector.prepare_data()\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/connectors/data_connector.py\", line 94, in prepare_data\r\n", + " call._call_lightning_datamodule_hook(trainer, \"prepare_data\")\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/lightning/pytorch/trainer/call.py\", line 164, in _call_lightning_datamodule_hook\r\n", + " return fn(*args, **kwargs)\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/src/data.py\", line 549, in prepare_data\r\n", + " prepare_data_static(**self._init_locals)\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/src/data.py\", line 101, in prepare_data_static\r\n", + " src_dataset = load_dataset(**load_dataset_params)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/load.py\", line 2136, in load_dataset\r\n", + " builder_instance.download_and_prepare(\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/builder.py\", line 954, in download_and_prepare\r\n", + " self._download_and_prepare(\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/builder.py\", line 1049, in _download_and_prepare\r\n", + " self._prepare_split(split_generator, **prepare_split_kwargs)\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/builder.py\", line 1842, in _prepare_split\r\n", + " for job_id, done, content in iflatmap_unordered(\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/utils/py_utils.py\", line 1387, in iflatmap_unordered\r\n", + " [async_result.get(timeout=0.05) for async_result in async_results]\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/datasets/utils/py_utils.py\", line 1387, in \r\n", + " [async_result.get(timeout=0.05) for async_result in async_results]\r\n", + " File \"/usr/local/lib/python3.10/dist-packages/multiprocess/pool.py\", line 774, in get\r\n", + " raise self._value\r\n", + "datasets.builder.DatasetGenerationError: An error occurred while generating the dataset\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish... \u001b[31m(failed 1).\u001b[0m Press Control-C to abort syncing.\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \\ 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: | 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: / 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)\r" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mv5r3-L6-D2048-E0.1 - Mem-Tune ctx-8k (train-ctx=4k, deepspeed_stage_1)\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments/runs/rjj8azcb\u001b[0m\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: ️⚡ View job at \u001b[34m\u001b[4mhttps://wandb.ai/rwkv-x-dev/RWKV-X-Experiments/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjkzMjg5ODA3/version_details/v52\u001b[0m\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\r\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20230914_002346-rjj8azcb/logs\u001b[0m\r\n" + ] + } + ], + "source": [ + "# Start the finetune model training\n", + "!cd \"{TRAINER_DIR}\" && \\\n", + " export WANDB_MODE=\"{WANDB_MODE}\" && \\\n", + " python3 lightning_trainer.py fit \\\n", + " -c \"{CONFIG_DIR}/config-mem-template.yaml\" \\\n", + " --trainer.logger.init_args.name=\"{WANDB_PREFIX} - Mem-Tune ctx-8k (train-ctx=4k, {DEEPSPEED_STRAT})\" \\\n", + " --trainer.strategy=\"{DEEPSPEED_STRAT}\" \\\n", + " --trainer.devices=\"{GPU_DEVICES}\" \\\n", + " --trainer.callbacks.init_args.dirpath=\"../checkpoint/{FILENAME_PREFIX}-mem-ctx-8k/\" \\\n", + " --model.lr_init=4e-4 \\\n", + " --model.lr_final=2e-4 \\\n", + " --data.max_token_size=8192 \\\n", + " --data.sort_by_length=True \\\n", + " --model.ctx_len=4096 \\\n", + " --model.bptt_learning_range=2 \\\n", + " --model.load_model=\"../model/{FILENAME_PREFIX}-mem-ctx-512.pth\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7c6294af", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:24:09.519443Z", + "iopub.status.busy": "2023-09-14T00:24:09.518429Z", + "iopub.status.idle": "2023-09-14T00:24:13.424492Z", + "shell.execute_reply": "2023-09-14T00:24:13.423305Z" + }, + "papermill": { + "duration": 3.942579, + "end_time": "2023-09-14T00:24:13.426912", + "exception": false, + "start_time": "2023-09-14T00:24:09.484333", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2023-09-14 00:24:11,902] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/export_checkpoint.py\", line 651, in \r\n", + " convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, output_file, save_dtype=args.dtype)\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/export_checkpoint.py\", line 542, in convert_zero_checkpoint_to_fp32_state_dict\r\n", + " state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)\r\n", + " File \"/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/RWKV-v5/export_checkpoint.py\", line 516, in get_fp32_state_dict_from_zero_checkpoint\r\n", + " raise ValueError(f\"Unable to find 'latest' file at {latest_path}\")\r\n", + "ValueError: Unable to find 'latest' file at ../checkpoint/v5r3-L6-D2048-E0_1-mem-ctx-8k/last.ckpt/latest\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ls: cannot access '../model/v5r3-L6-D2048-E0_1-mem-ctx-8k.pth': No such file or directory\r\n" + ] + } + ], + "source": [ + "# Lets export the model from the checkpoint\n", + "!cd \"{TRAINER_DIR}\" && \\\n", + " python3 export_checkpoint.py \\\n", + " \"../checkpoint/{FILENAME_PREFIX}-mem-ctx-8k/last.ckpt\" \\\n", + " \"../model/{FILENAME_PREFIX}-mem-ctx-8k.pth\" \"bf16\"\n", + "!cd \"{TRAINER_DIR}\" && ls -alh \"../model/{FILENAME_PREFIX}-mem-ctx-8k.pth\"" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "959cb71d", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:24:13.493646Z", + "iopub.status.busy": "2023-09-14T00:24:13.493200Z", + "iopub.status.idle": "2023-09-14T00:24:13.760260Z", + "shell.execute_reply": "2023-09-14T00:24:13.759053Z" + }, + "papermill": { + "duration": 0.303638, + "end_time": "2023-09-14T00:24:13.762669", + "exception": false, + "start_time": "2023-09-14T00:24:13.459031", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "python3: can't open file '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/../memory_script/eval_v5_memory_guided.py': [Errno 2] No such file or directory\r\n" + ] + } + ], + "source": [ + "# Lets do a quick memory test\n", + "!python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-mem-ctx-8k.pth\"" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "c39913f9", + "metadata": { + "execution": { + "iopub.execute_input": "2023-09-14T00:24:13.829059Z", + "iopub.status.busy": "2023-09-14T00:24:13.828546Z", + "iopub.status.idle": "2023-09-14T00:24:14.094937Z", + "shell.execute_reply": "2023-09-14T00:24:14.093719Z" + }, + "papermill": { + "duration": 0.302801, + "end_time": "2023-09-14T00:24:14.097341", + "exception": false, + "start_time": "2023-09-14T00:24:13.794540", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "python3: can't open file '/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/../memory_script/eval_v5_memory_guided.py': [Errno 2] No such file or directory\r\n" + ] + } + ], + "source": [ + "!python3 ../memory_script/eval_v5_memory_guided.py \"{PROJECT_DIR}/model/{FILENAME_PREFIX}-mem-ctx-8k.pth\" \"none\" 1000 4000" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python3 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + }, + "papermill": { + "default_parameters": {}, + "duration": 87.06318, + "end_time": "2023-09-14T00:24:14.449806", + "environment_variables": {}, + "exception": null, + "input_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/notebook/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/stage3.ipynb", + "output_path": "/actions-runner/_work/RWKV-infctx-trainer/RWKV-infctx-trainer/output/experiment/rwkv-x-exp/v5-r3-memory/L6-D2048-E1e-1-ctx4k/stage3.ipynb", + "parameters": {}, + "start_time": "2023-09-14T00:22:47.386626", + "version": "2.4.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file