data wrangling notebook, to add base64 images to the parquet itself

_dev/add_videos_to_dataset.ipynb

@@ -0,0 +1,538 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "One off conversion / data wrangling script, to avoid redownloading videos."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import base64\n",
+    "\n",
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_parquet(\"../data/dataset_original.parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>video_id</th>\n",
+       "      <th>frame_idx</th>\n",
+       "      <th>timestamp</th>\n",
+       "      <th>image_path</th>\n",
+       "      <th>dim_0</th>\n",
+       "      <th>dim_1</th>\n",
+       "      <th>dim_2</th>\n",
+       "      <th>dim_3</th>\n",
+       "      <th>dim_4</th>\n",
+       "      <th>dim_5</th>\n",
+       "      <th>...</th>\n",
+       "      <th>dim_502</th>\n",
+       "      <th>dim_503</th>\n",
+       "      <th>dim_504</th>\n",
+       "      <th>dim_505</th>\n",
+       "      <th>dim_506</th>\n",
+       "      <th>dim_507</th>\n",
+       "      <th>dim_508</th>\n",
+       "      <th>dim_509</th>\n",
+       "      <th>dim_510</th>\n",
+       "      <th>dim_511</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>data/images/8Ilh1ewceco/0.jpg</td>\n",
+       "      <td>-0.013127</td>\n",
+       "      <td>-0.022996</td>\n",
+       "      <td>-0.049374</td>\n",
+       "      <td>-0.006306</td>\n",
+       "      <td>0.013602</td>\n",
+       "      <td>-0.003762</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.006920</td>\n",
+       "      <td>0.013831</td>\n",
+       "      <td>0.056647</td>\n",
+       "      <td>0.007946</td>\n",
+       "      <td>-0.002478</td>\n",
+       "      <td>-0.030497</td>\n",
+       "      <td>-0.011770</td>\n",
+       "      <td>0.067427</td>\n",
+       "      <td>-0.031810</td>\n",
+       "      <td>-0.025615</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>145</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>data/images/8Ilh1ewceco/145.jpg</td>\n",
+       "      <td>0.009040</td>\n",
+       "      <td>0.003338</td>\n",
+       "      <td>0.029684</td>\n",
+       "      <td>-0.033058</td>\n",
+       "      <td>0.040864</td>\n",
+       "      <td>-0.006447</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.033575</td>\n",
+       "      <td>-0.019076</td>\n",
+       "      <td>0.047166</td>\n",
+       "      <td>-0.010574</td>\n",
+       "      <td>-0.018608</td>\n",
+       "      <td>-0.013465</td>\n",
+       "      <td>-0.020017</td>\n",
+       "      <td>0.086240</td>\n",
+       "      <td>-0.029653</td>\n",
+       "      <td>0.035949</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>290</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>data/images/8Ilh1ewceco/290.jpg</td>\n",
+       "      <td>0.004891</td>\n",
+       "      <td>0.006527</td>\n",
+       "      <td>0.004417</td>\n",
+       "      <td>-0.000323</td>\n",
+       "      <td>0.006400</td>\n",
+       "      <td>-0.024191</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.043122</td>\n",
+       "      <td>-0.010695</td>\n",
+       "      <td>0.005672</td>\n",
+       "      <td>0.000172</td>\n",
+       "      <td>-0.014442</td>\n",
+       "      <td>-0.014647</td>\n",
+       "      <td>-0.016840</td>\n",
+       "      <td>0.100285</td>\n",
+       "      <td>0.013794</td>\n",
+       "      <td>0.015046</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>435</td>\n",
+       "      <td>15.0</td>\n",
+       "      <td>data/images/8Ilh1ewceco/435.jpg</td>\n",
+       "      <td>-0.022159</td>\n",
+       "      <td>0.020703</td>\n",
+       "      <td>-0.021607</td>\n",
+       "      <td>-0.019721</td>\n",
+       "      <td>-0.006067</td>\n",
+       "      <td>-0.035070</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.017047</td>\n",
+       "      <td>-0.018341</td>\n",
+       "      <td>-0.006733</td>\n",
+       "      <td>-0.007040</td>\n",
+       "      <td>-0.008368</td>\n",
+       "      <td>0.009755</td>\n",
+       "      <td>-0.045662</td>\n",
+       "      <td>0.116601</td>\n",
+       "      <td>-0.000572</td>\n",
+       "      <td>-0.000985</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>580</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>data/images/8Ilh1ewceco/580.jpg</td>\n",
+       "      <td>-0.015903</td>\n",
+       "      <td>0.033545</td>\n",
+       "      <td>0.009257</td>\n",
+       "      <td>-0.033540</td>\n",
+       "      <td>0.010586</td>\n",
+       "      <td>-0.028067</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.016532</td>\n",
+       "      <td>0.012388</td>\n",
+       "      <td>0.020868</td>\n",
+       "      <td>-0.012635</td>\n",
+       "      <td>0.010914</td>\n",
+       "      <td>0.009203</td>\n",
+       "      <td>-0.010078</td>\n",
+       "      <td>0.063971</td>\n",
+       "      <td>-0.038024</td>\n",
+       "      <td>0.025840</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 516 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      video_id  frame_idx  timestamp                       image_path  \\\n",
+       "0  8Ilh1ewceco          0        0.0    data/images/8Ilh1ewceco/0.jpg   \n",
+       "1  8Ilh1ewceco        145        5.0  data/images/8Ilh1ewceco/145.jpg   \n",
+       "2  8Ilh1ewceco        290       10.0  data/images/8Ilh1ewceco/290.jpg   \n",
+       "3  8Ilh1ewceco        435       15.0  data/images/8Ilh1ewceco/435.jpg   \n",
+       "4  8Ilh1ewceco        580       20.0  data/images/8Ilh1ewceco/580.jpg   \n",
+       "\n",
+       "      dim_0     dim_1     dim_2     dim_3     dim_4     dim_5  ...   dim_502  \\\n",
+       "0 -0.013127 -0.022996 -0.049374 -0.006306  0.013602 -0.003762  ... -0.006920   \n",
+       "1  0.009040  0.003338  0.029684 -0.033058  0.040864 -0.006447  ...  0.033575   \n",
+       "2  0.004891  0.006527  0.004417 -0.000323  0.006400 -0.024191  ... -0.043122   \n",
+       "3 -0.022159  0.020703 -0.021607 -0.019721 -0.006067 -0.035070  ... -0.017047   \n",
+       "4 -0.015903  0.033545  0.009257 -0.033540  0.010586 -0.028067  ... -0.016532   \n",
+       "\n",
+       "    dim_503   dim_504   dim_505   dim_506   dim_507   dim_508   dim_509  \\\n",
+       "0  0.013831  0.056647  0.007946 -0.002478 -0.030497 -0.011770  0.067427   \n",
+       "1 -0.019076  0.047166 -0.010574 -0.018608 -0.013465 -0.020017  0.086240   \n",
+       "2 -0.010695  0.005672  0.000172 -0.014442 -0.014647 -0.016840  0.100285   \n",
+       "3 -0.018341 -0.006733 -0.007040 -0.008368  0.009755 -0.045662  0.116601   \n",
+       "4  0.012388  0.020868 -0.012635  0.010914  0.009203 -0.010078  0.063971   \n",
+       "\n",
+       "    dim_510   dim_511  \n",
+       "0 -0.031810 -0.025615  \n",
+       "1 -0.029653  0.035949  \n",
+       "2  0.013794  0.015046  \n",
+       "3 -0.000572 -0.000985  \n",
+       "4 -0.038024  0.025840  \n",
+       "\n",
+       "[5 rows x 516 columns]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 71761/71761 [00:49<00:00, 1458.75it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "image_paths = df[\"image_path\"].tolist()\n",
+    "new_df = df.rename(columns={'image_path': 'base64_image'})\n",
+    "for i, img in enumerate(tqdm(image_paths)):\n",
+    "    with open(f\"../{img}\", \"rb\") as image_file:\n",
+    "        encoded_string = base64.b64encode(image_file.read()).decode()\n",
+    "    new_df.loc[i, 'base64_image'] = encoded_string"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/sidneyradcliffe/miniforge3/envs/visual-content-search-over-videos/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pipeline.process_videos import DATAFRAME_PATH\n",
+    "new_df.to_parquet(DATAFRAME_PATH, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# reload, check it's correct\n",
+    "new_df = pd.read_parquet(DATAFRAME_PATH)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>video_id</th>\n",
+       "      <th>frame_idx</th>\n",
+       "      <th>timestamp</th>\n",
+       "      <th>base64_image</th>\n",
+       "      <th>dim_0</th>\n",
+       "      <th>dim_1</th>\n",
+       "      <th>dim_2</th>\n",
+       "      <th>dim_3</th>\n",
+       "      <th>dim_4</th>\n",
+       "      <th>dim_5</th>\n",
+       "      <th>...</th>\n",
+       "      <th>dim_502</th>\n",
+       "      <th>dim_503</th>\n",
+       "      <th>dim_504</th>\n",
+       "      <th>dim_505</th>\n",
+       "      <th>dim_506</th>\n",
+       "      <th>dim_507</th>\n",
+       "      <th>dim_508</th>\n",
+       "      <th>dim_509</th>\n",
+       "      <th>dim_510</th>\n",
+       "      <th>dim_511</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...</td>\n",
+       "      <td>-0.013127</td>\n",
+       "      <td>-0.022996</td>\n",
+       "      <td>-0.049374</td>\n",
+       "      <td>-0.006306</td>\n",
+       "      <td>0.013602</td>\n",
+       "      <td>-0.003762</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.006920</td>\n",
+       "      <td>0.013831</td>\n",
+       "      <td>0.056647</td>\n",
+       "      <td>0.007946</td>\n",
+       "      <td>-0.002478</td>\n",
+       "      <td>-0.030497</td>\n",
+       "      <td>-0.011770</td>\n",
+       "      <td>0.067427</td>\n",
+       "      <td>-0.031810</td>\n",
+       "      <td>-0.025615</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>145</td>\n",
+       "      <td>5.0</td>\n",
+       "      <td>b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...</td>\n",
+       "      <td>0.009040</td>\n",
+       "      <td>0.003338</td>\n",
+       "      <td>0.029684</td>\n",
+       "      <td>-0.033058</td>\n",
+       "      <td>0.040864</td>\n",
+       "      <td>-0.006447</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.033575</td>\n",
+       "      <td>-0.019076</td>\n",
+       "      <td>0.047166</td>\n",
+       "      <td>-0.010574</td>\n",
+       "      <td>-0.018608</td>\n",
+       "      <td>-0.013465</td>\n",
+       "      <td>-0.020017</td>\n",
+       "      <td>0.086240</td>\n",
+       "      <td>-0.029653</td>\n",
+       "      <td>0.035949</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>290</td>\n",
+       "      <td>10.0</td>\n",
+       "      <td>b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...</td>\n",
+       "      <td>0.004891</td>\n",
+       "      <td>0.006527</td>\n",
+       "      <td>0.004417</td>\n",
+       "      <td>-0.000323</td>\n",
+       "      <td>0.006400</td>\n",
+       "      <td>-0.024191</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.043122</td>\n",
+       "      <td>-0.010695</td>\n",
+       "      <td>0.005672</td>\n",
+       "      <td>0.000172</td>\n",
+       "      <td>-0.014442</td>\n",
+       "      <td>-0.014647</td>\n",
+       "      <td>-0.016840</td>\n",
+       "      <td>0.100285</td>\n",
+       "      <td>0.013794</td>\n",
+       "      <td>0.015046</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>435</td>\n",
+       "      <td>15.0</td>\n",
+       "      <td>b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...</td>\n",
+       "      <td>-0.022159</td>\n",
+       "      <td>0.020703</td>\n",
+       "      <td>-0.021607</td>\n",
+       "      <td>-0.019721</td>\n",
+       "      <td>-0.006067</td>\n",
+       "      <td>-0.035070</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.017047</td>\n",
+       "      <td>-0.018341</td>\n",
+       "      <td>-0.006733</td>\n",
+       "      <td>-0.007040</td>\n",
+       "      <td>-0.008368</td>\n",
+       "      <td>0.009755</td>\n",
+       "      <td>-0.045662</td>\n",
+       "      <td>0.116601</td>\n",
+       "      <td>-0.000572</td>\n",
+       "      <td>-0.000985</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>8Ilh1ewceco</td>\n",
+       "      <td>580</td>\n",
+       "      <td>20.0</td>\n",
+       "      <td>b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...</td>\n",
+       "      <td>-0.015903</td>\n",
+       "      <td>0.033545</td>\n",
+       "      <td>0.009257</td>\n",
+       "      <td>-0.033540</td>\n",
+       "      <td>0.010586</td>\n",
+       "      <td>-0.028067</td>\n",
+       "      <td>...</td>\n",
+       "      <td>-0.016532</td>\n",
+       "      <td>0.012388</td>\n",
+       "      <td>0.020868</td>\n",
+       "      <td>-0.012635</td>\n",
+       "      <td>0.010914</td>\n",
+       "      <td>0.009203</td>\n",
+       "      <td>-0.010078</td>\n",
+       "      <td>0.063971</td>\n",
+       "      <td>-0.038024</td>\n",
+       "      <td>0.025840</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 516 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      video_id  frame_idx  timestamp  \\\n",
+       "0  8Ilh1ewceco          0        0.0   \n",
+       "1  8Ilh1ewceco        145        5.0   \n",
+       "2  8Ilh1ewceco        290       10.0   \n",
+       "3  8Ilh1ewceco        435       15.0   \n",
+       "4  8Ilh1ewceco        580       20.0   \n",
+       "\n",
+       "                                        base64_image     dim_0     dim_1  \\\n",
+       "0  b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH... -0.013127 -0.022996   \n",
+       "1  b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...  0.009040  0.003338   \n",
+       "2  b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH...  0.004891  0.006527   \n",
+       "3  b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH... -0.022159  0.020703   \n",
+       "4  b'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgH... -0.015903  0.033545   \n",
+       "\n",
+       "      dim_2     dim_3     dim_4     dim_5  ...   dim_502   dim_503   dim_504  \\\n",
+       "0 -0.049374 -0.006306  0.013602 -0.003762  ... -0.006920  0.013831  0.056647   \n",
+       "1  0.029684 -0.033058  0.040864 -0.006447  ...  0.033575 -0.019076  0.047166   \n",
+       "2  0.004417 -0.000323  0.006400 -0.024191  ... -0.043122 -0.010695  0.005672   \n",
+       "3 -0.021607 -0.019721 -0.006067 -0.035070  ... -0.017047 -0.018341 -0.006733   \n",
+       "4  0.009257 -0.033540  0.010586 -0.028067  ... -0.016532  0.012388  0.020868   \n",
+       "\n",
+       "    dim_505   dim_506   dim_507   dim_508   dim_509   dim_510   dim_511  \n",
+       "0  0.007946 -0.002478 -0.030497 -0.011770  0.067427 -0.031810 -0.025615  \n",
+       "1 -0.010574 -0.018608 -0.013465 -0.020017  0.086240 -0.029653  0.035949  \n",
+       "2  0.000172 -0.014442 -0.014647 -0.016840  0.100285  0.013794  0.015046  \n",
+       "3 -0.007040 -0.008368  0.009755 -0.045662  0.116601 -0.000572 -0.000985  \n",
+       "4 -0.012635  0.010914  0.009203 -0.010078  0.063971 -0.038024  0.025840  \n",
+       "\n",
+       "[5 rows x 516 columns]"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "new_df.head()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "visual-content-search-over-videos",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}