{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8474f311-77f5-4063-be0e-cf2ae3b4a16e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e03105efb444ac9934a1046e787b03c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/40.0k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "080310760014445f82980ad66376301e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.model:   0%|          | 0.00/4.24M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a1a0f0f541549ee9142ada5bbc955a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/17.5M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67777175bcce4a9ba1a487a226636178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/636 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "datapath = 'readsy/stories/'\n",
    "pairpath = 'readsy/pairs/readsy_story_pairs0407.csv'\n",
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"unsloth/gemma-2b-bnb-4bit\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26525a98-8421-4415-87be-ad099f708e4e",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'dataloader'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#test dataloader \u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdataloader\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m StoryPairDataset\n\u001b[1;32m      3\u001b[0m dataloader \u001b[38;5;241m=\u001b[39m StoryPairDataset(datapath,\n\u001b[1;32m      4\u001b[0m                               pairpath,\n\u001b[1;32m      5\u001b[0m                               tokenizer,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     13\u001b[0m                               least_likes\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m,\n\u001b[1;32m     14\u001b[0m                               margin\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'dataloader'"
     ]
    }
   ],
   "source": [
    "#test dataloader \n",
    "from dataloader import StoryPairDataset\n",
    "dataloader = StoryPairDataset(datapath,\n",
    "                              pairpath,\n",
    "                              tokenizer,\n",
    "                              task='rm',\n",
    "                              used_dataset_size=100,\n",
    "                              train_test_split=0.1,\n",
    "                              split_by='random',\n",
    "                              max_len=1024*4,\n",
    "                              mode='m2',\n",
    "                              max_time_window=5400,\n",
    "                              least_likes=5,\n",
    "                              margin=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2afde46f-c120-4651-b353-50fa898b529f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0117_2story_pairs.csv  crawlReedsy.py  prompt.csv  unused_prompts.csv\n",
      "Untitled.ipynb         \u001b[0m\u001b[01;34mpairs\u001b[0m/          \u001b[01;34mstories\u001b[0m/\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57a7b6ce-1638-49c0-acfd-30e34d9e17bb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}