Add an evluation script

run_medclip.sh

@@ -1,8 +1,8 @@
 python src/hybrid_clip/run_hybrid_clip.py \
-    --output_dir ./snapshots/vision_augmented \
-    --text_model_name_or_path="roberta-base" \
+    --output_dir ./snapshots/vision_augmented_biobert \
+    --text_model_name_or_path="allenai/scibert_scivocab_uncased" \
     --vision_model_name_or_path="openai/clip-vit-base-patch32" \
-    --tokenizer_name="roberta-base" \
+    --tokenizer_name="allenai/scibert_scivocab_uncased" \
     --train_file="data/train_dataset.json" \
     --validation_file="data/valid_dataset.json" \
     --do_train --do_eval \

src/hybrid_clip/test_clip.ipynb

@@ -26,77 +26,85 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
+   "execution_count": 38,
    "source": [
+    "from transformers import AutoTokenizer\n",
+    "from configuration_hybrid_clip import HybridCLIPConfig\n",
     "from modeling_hybrid_clip import FlaxHybridCLIP\n",
-    "import jax\n"
-   ]
+    "  \n",
+    "# config = HybridCLIPConfig.from_pretrained(\"../..\")\n",
+    "model = FlaxHybridCLIP.from_pretrained(\"flax-community/medclip-roco\")"
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stderr",
-     "text": [
-      "INFO:absl:Starting the local TPU driver.\n",
-      "INFO:absl:Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://\n",
-      "INFO:absl:Unable to initialize backend 'tpu': Invalid argument: TpuPlatform is not available.\n"
-     ]
-    }
-   ],
+   "execution_count": 39,
    "source": [
-    "model = FlaxHybridCLIP.from_text_vision_pretrained(\"bert-base-uncased\", \"openai/clip-vit-base-patch32\")"
-   ]
+    "tokenizer = AutoTokenizer.from_pretrained(\"allenai/scibert_scivocab_uncased\")"
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
+   "execution_count": 31,
+   "source": [
+    "import torch\n",
+    "import numpy as np\n",
+    "from run_hybrid_clip import Transform\n",
+    "from torchvision.transforms.functional import InterpolationMode\n",
+    "\n",
+    "image_size = 224\n",
+    "transforms = Transform(image_size)\n"
+   ],
    "outputs": [],
+   "metadata": {}
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
    "source": [
-    "model = FlaxHybridCLIP.from_text_vision_pretrained(\"bert-base-uncased\", \"openai/clip-vit-base-patch32\", text_from_pt=True, vision_from_pt=True)"
-   ]
+    "from torchvision.io.image import read_image, ImageReadMode\n",
+    "\n",
+    "# TODO create a batch of images\n",
+    "img = read_image('../../data/PMC4917066_amjcaserep-17-301-g001.jpg', mode=ImageReadMode.RGB)\n",
+    "tr_img = transforms(img)\n"
+   ],
+   "outputs": [],
+   "metadata": {}
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
+   "execution_count": 37,
+   "source": [
+    "max_seq_length = 128\n",
+    "pixel_values = torch.stack([example[0] for example in examples]).permute(0, 2, 3, 1).numpy()\n",
+    "# pixel_values = torch.stack([example[0] for example in examples]).numpy()\n",
+    "captions = [example[1] for example in examples]\n",
+    "inputs = tokenizer(captions, max_length=max_seq_length, padding=\"max_length\", return_tensors=\"np\",\n",
+    "                    truncation=True)\n",
+    "batch = {\n",
+    "        \"pixel_values\": pixel_values,\n",
+    "        \"input_ids\": inputs[\"input_ids\"],\n",
+    "        \"attention_mask\": inputs[\"attention_mask\"],\n",
+    "    }\n",
+    "logits = model(**batch, train=False)[0]"
+   ],
    "outputs": [
-    {
-     "output_type": "stream",
-     "name": "stderr",
-     "text": [
-      "INFO:absl:Starting the local TPU driver.\n",
-      "INFO:absl:Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://\n",
-      "INFO:absl:Unable to initialize backend 'tpu': Invalid argument: TpuPlatform is not available.\n"
-     ]
-    },
     {
      "output_type": "execute_result",
      "data": {
       "text/plain": [
-       "[GpuDevice(id=0, process_index=0)]"
+       "torch.Size([3, 224, 224])"
       ]
      },
      "metadata": {},
-     "execution_count": 3
+     "execution_count": 37
     }
    ],
-   "source": [
-    "jax.devices()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
+   "metadata": {}
   }
  ]
 }