{ "cells": [ { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "import fastbook\n", "fastbook.setup_book()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from fastbook import *\n", "from fastai.vision.widgets import *\n", "import skimage\n", "from skimage import io as skio\n", "import numpy\n", "from PIL import Image, ImageEnhance" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "path =Path(\"D:/pytorch/data/2D_FishProcessor\")\n", "Path.BASE_PATH =path\n", "def name_to_hrs (r): return float(round(float(os.path.basename(r)[0:-4].split(\"_\")[1][1:])*(minutes/60)+5,2))\n", "def validation_split (r): return os.path.basename(r)[0:-4].split(\"_\")[3] == \"R0003\" or os.path.basename(r)[0:-4].split(\"_\")[3] == \"R0006\"\n", "def get_label_filename(name): return path/'labels'/f'{name.stem}_annotationLabels.tif'\n" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "\n", "out_pl = widgets.Output()\n", "out_pl_mask = widgets.Output()\n", "lbl_pred = widgets.Label()\n", "zebrafish_age_predictor = load_learner(path/'FishAge.pkl')\n", "zebrafish_classifier = load_learner(path/'FishSegmentation.pkl')\n", "btn_run = widgets.Button(description='Process')\n", "def on_click_classify(change):\n", " img = PILImage.create(btn_upload.data[-1])\n", " out_pl.clear_output()\n", " enhancer = ImageEnhance.Brightness(img)\n", " factor = 10 \n", " im_output = enhancer.enhance(factor)\n", " with out_pl: display(im_output.to_thumb(256,256))\n", " age,tensor, tensor=zebrafish_age_predictor.predict(img)\n", " \n", " pred,pred_idx,probs=zebrafish_classifier.predict(img)\n", " with out_pl_mask: pred.show(alpha=1, vmin=0, vmax=3, title='mask')\n", " lbl_pred.value = f'Predicted age: {age[0]};'\n", "btn_run.on_click(on_click_classify)\n" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "btn_upload= widgets.FileUpload()" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f6257adb42bc459facb3447be0ebc83d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(Label(value='Upload a grayscale zebrafish embryo image'), FileUpload(value={}, description='Upl…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR4AAAEvCAYAAACT767eAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAALFklEQVR4nO3dzYtkZxXA4XNCjBgMxICudJIokREJBkFXkVlIhCELhyCziLpQURAJMjJIiCgBDWQxEPwDBHfCbJyFw/iBmFazENxEiRlBnGRA1Bgx4ipEc110d1LdU9VdH7dO3brv82xCd9VUqqve+tW599btzq7rAqDSTZu+A0B7hAcoJzxAOeEBygkPUE54gHLCQ4nMfDwz/7Tp+8EwCA9QTniAcsLToMx8OjO/l5nfycyXMvOVzHwiM2/KzG9l5t8z8x+Z+cTEv3k4M3+Tmf/OzJcz83Jmvv/Q7T6WmX/OzFf3/v1PMvNtM+7DHZn5TGbuZObta/6RGRjhadenIuItEXF/RHwtIh6LiB9FxNsj4mMRcT4iHsvM03vXf2tEfDsiPhwRD0TE/yLicmbeEhGRmQ9FxKMR8dWIuGfvOlem/Y8z80RE/Doi/hoRn+i67pX+fzyGLJ2r1Z7MfDoibu+67r6J7z0XEa93XXfvxPeejYifdV13fspt3BER/4yI+7uueyYzz0XElyPig13XvTbl+o9HxGci4qHYDdKliHik67rX+/vJ2BYmnnY9e+jrv0XE76Z8710REZl5X2b+MDOvZeZ/IuL63nXu3PvvxdidoF7MzO9n5mcz87ZDt/fOiPhlRPyg67qviE67hKddh6eSbsb3bsrMWyPip3tffz4iPhoRH9n7+paIiK7r/hIRJ/cufykivhkRf8zM90zc3isR8auI+GRmvrvPH4btIjzM4wOxO618o+u6X3Rd93xEvCMicvJKXde92nXdj7uu+3pE3BsRt0bEmYmrvBa7m1q/j4idzLwzaJLwMI8XI+LViHgkM9+XmR+PiO/G7sQTERGZ+YXM/GJmfmgvKJ+OiNsi4g+TN7S3/+dsRPw2duPz3qofguEQHo7Vdd3Lsbtj+IGIeC4iLsTuUa/JfTT/iojPRcTTEfF87B4p+1LXdT+fcnv/jYiHY/fI1k5m3rPO+8/wOKoFlDPxAOWEBygnPEA54QHKCQ9Q7uajLrzr0csOeQFLeeHJB3PWZSYeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOVu3vQdoF1XLp2/4Xunz1zYwD2hmvCwFtOiss5/FyFa20R4WMkqoejb4fsiRMMlPCxkSKE5jhANl/BwrG2KzVGEaDiEh6nGEpujXLl0Xnw2RHh4QwuxOWzyZxahOsLTuBZjM4sI1RGeRgnO0fYfHwFaD+FpjOAsRoDWwykTDRGd5V25dN7j1yMTTwO8YPpjAuqHiWfkRGc9PK6rMfGMkBdFDdPP8kw8IyM69ez/WZyJZyQs/Pnd/dTFA19fO3e2l9s1Ac0vu66beeFdj16efSGDITrzOxydw/qKkPhEvPDkgznrMptaW0505ndcdOa9zjxsfh1NeLaYhb0efcWH2YRnS4nOet391MVeAmTymc7O5S1kIdeZFZ9F9wX5FRwH2bm8RQRneevafFpmZ3QrAbJzeQREZ3nr3GezzG17LoVnK1iow7bM/qDWn1PhGbjWF+iqKo9QLRqglp9b4RmwlhdmHzZ1WFx8jic8A9XqguxDX4fCV70PzCY8jMo2vuBbfJNxOH1gWlyEfRhqcBY53D62w+xHHU73AcIBaTU6q5y4OdTgLKOls9tNPAPRYnTGFI2jtPohQx8gHDjRGTcfMryR8GzY2BfYNC1FZ5/4HCQ8lGoxOvvE503Cs0FjXVSztBydfR6DXcIDxZzXJTwbM8bFdBTv9Ae1/ngIzwaIDhFtn9MlPMXGtoCOIzr9GdPaEZ5CY1o48xCd4816jE6fuTD1Q4RjWUPCU2QsC4b+TcbncHCmBWgMa8m5WgXGsFBYr7ufuhgnd67PvHw/PmNZS8IDA3BUdCaN4RyuCJtaazeWd6hlXDt3trc/Ccy4CA9QTnjWqOVpZ5Kp52jzbmaNifBQQnymazE6EcKzNqadG4nPQa1GJ0J4KCY+u1qOToTwrIVp52itx6f16EQIDxvSenxaJzw9M+3MT3zaJTxslPi0SXh6ZNpZTmvxuXrqxKbvwsYJD2xA6/ERnp6YdlbT2tTTOuFhMFqLT8tTj/DABrUaH+Hpgc0sWIzwMCitbW5FtDn1CA9QTnhWZDMLFic8DE6Lm1ut8cveWbvDfztKWBAeejPvH/Cbdr39GPkjgG0QnhXYv9NfKASnLcLD3MSBvggPxxKc9bt66kRTv5lQeJhKbFgnh9OXZP8OLM/EwwEmHSqYeIBywgOUE54ljHX/js2szWrpLHXhAcoJD1DOUS0iwmbWprX04cEIEw9sXGvRiRAeYAOEB5tZG9TitBMhPM0THTZBeBomOmyK8DRKdNgk4WmQ6LBpwtMY0RmOVncsR/gAYTMEhyEx8TRAdBgaE8+ICc5wtbyZFSE8oyQ4w9Z6dCKEZ+uJzPYQnDcJzxJOn7lQ8svARGU8ROcgO5cHSnTGQ3RuZOIZINEZB8GZzcQzMKIzDqJzNOEZENEZB9E5nvAA5YRnSafPXOj19kw7tER4gHLCMwCmnfGwf2c+wrOCPoIhOrRIeFZwcue6cMAShKcHy8ZHtGiV8PRk0YiIDi0TnhVN7kwUE5iP8PRsnvgIFK1zkmgPTu5cj6unTrzx9WRYrp07e+C6ogPCs3ZCAzeyqdUTHxyD+QkPUE54enRy57rJB+YgPNATbzrzE541sADhaMIDlBOeNTH1tMXzvRjhWSOLEaYTnjUTn/HzHC9OeApYmHCQ8MAKvKksx7laRfYX6OTJpAyTmKyfiaeYTzcPm+emhvBsiAU+PJ6TOsKzQRb6cHguagnPhlnwm+c5qCc8AzDvwt/fP+SFwrZzVGsgjjrqNS00jpKxzUw8A3N4ojluujH9sI2EZ6AW2aQSH7aN8ADlhAeWZP/a8oQHliA6qxEemrdKRARoOcIzAhb/6hZ5DD3eqxOeLXb11AkvggHwHCxOeLaYw+j9micgs64jPosRni0nPv06KiCzLnMay+KcMjECJ3eue8ft0f5jedzjKjbLE56RcO5W/46acFiNTa2R8aJYL49vP0w8I2T66ZfY9E94RmzyBSNCixGb9RKeRojQfASnhvA0aGgRmvZir7xfYlNPeBp3+EU3lBd8xUcEBGdzHNXigKrf6zzP7a/zPojOZpl4mGld09AiL/q+Jx/BGQbhYW6r7otZ9kXf18cDRGc4hIeVHBeFPl/sqwRIdIZFeOhF5Qt70QCJzvAID1trno8FiM4wCQ+jcHgKEpxhEx5GRXC2g8/xAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniAcsIDlBMeoJzwAOWEBygnPEA54QHKCQ9QTniActl13abvA9AYEw9QTniAcsIDlBMeoJzwAOWEByj3fzGDL4/jh51rAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "VBox([widgets.Label('Upload a grayscale zebrafish embryo image'), \n", " btn_upload, btn_run, out_pl, out_pl_mask, lbl_pred])" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting voila\n", " Downloading voila-0.3.5-py3-none-any.whl (1.7 MB)\n", "Collecting jupyterlab-server<3,>=2.3.0\n", " Downloading jupyterlab_server-2.14.0-py3-none-any.whl (54 kB)\n", "Requirement already satisfied: traitlets<6,>=5.0.3 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from voila) (5.1.1)\n", "Collecting jupyter-server<2.0.0,>=0.3.0\n", " Downloading jupyter_server-1.17.1-py3-none-any.whl (344 kB)\n", "Requirement already satisfied: nbclient<0.6,>=0.4.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from voila) (0.5.13)\n", "Requirement already satisfied: jupyter-client<8,>=6.1.3 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from voila) (7.2.2)\n", "Collecting nbconvert<7,>=6.4.5\n", " Using cached nbconvert-6.5.0-py3-none-any.whl (561 kB)\n", "Collecting websockets>=9.0\n", " Downloading websockets-10.3-cp310-cp310-win_amd64.whl (98 kB)\n", "Requirement already satisfied: jupyter-core>=4.9.2 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (4.10.0)\n", "Requirement already satisfied: nest-asyncio>=1.5.4 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (1.5.5)\n", "Requirement already satisfied: pyzmq>=22.3 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (22.3.0)\n", "Requirement already satisfied: entrypoints in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (0.4)\n", "Requirement already satisfied: tornado>=6.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (6.1)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-client<8,>=6.1.3->voila) (2.8.2)\n", "Requirement already satisfied: pywin32>=1.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-core>=4.9.2->jupyter-client<8,>=6.1.3->voila) (302)\n", "Requirement already satisfied: argon2-cffi in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (21.3.0)\n", "Requirement already satisfied: nbformat>=5.2.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (5.3.0)\n", "Collecting websocket-client\n", " Downloading websocket_client-1.3.3-py3-none-any.whl (54 kB)\n", "Requirement already satisfied: jinja2 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (3.0.3)\n", "Requirement already satisfied: terminado>=0.8.3 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (0.13.1)\n", "Collecting anyio<4,>=3.1.0\n", " Downloading anyio-3.6.1-py3-none-any.whl (80 kB)\n", "Requirement already satisfied: packaging in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (21.3)\n", "Requirement already satisfied: prometheus-client in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (0.13.1)\n", "Requirement already satisfied: Send2Trash in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (1.8.0)\n", "Requirement already satisfied: pywinpty in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyter-server<2.0.0,>=0.3.0->voila) (2.0.2)\n", "Collecting sniffio>=1.1\n", " Downloading sniffio-1.2.0-py3-none-any.whl (10 kB)\n", "Requirement already satisfied: idna>=2.8 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from anyio<4,>=3.1.0->jupyter-server<2.0.0,>=0.3.0->voila) (3.3)\n", "Requirement already satisfied: requests in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyterlab-server<3,>=2.3.0->voila) (2.27.1)\n", "Requirement already satisfied: jsonschema>=3.0.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jupyterlab-server<3,>=2.3.0->voila) (4.4.0)\n", "Collecting babel\n", " Downloading Babel-2.10.3-py3-none-any.whl (9.5 MB)\n", "Collecting json5\n", " Downloading json5-0.9.8.tar.gz (22 kB)\n", "Requirement already satisfied: MarkupSafe>=2.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jinja2->jupyter-server<2.0.0,>=0.3.0->voila) (2.0.1)\n", "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jsonschema>=3.0.1->jupyterlab-server<3,>=2.3.0->voila) (0.18.0)\n", "Requirement already satisfied: attrs>=17.4.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from jsonschema>=3.0.1->jupyterlab-server<3,>=2.3.0->voila) (21.4.0)\n", "Requirement already satisfied: bleach in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (4.1.0)\n", "Requirement already satisfied: defusedxml in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (0.7.1)\n", "Requirement already satisfied: pandocfilters>=1.4.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (1.5.0)\n", "Requirement already satisfied: pygments>=2.4.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (2.11.2)\n", "Collecting tinycss2\n", " Using cached tinycss2-1.1.1-py3-none-any.whl (21 kB)\n", "Requirement already satisfied: beautifulsoup4 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (4.11.1)\n", "Requirement already satisfied: mistune<2,>=0.8.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (0.8.4)\n", "Requirement already satisfied: jupyterlab-pygments in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbconvert<7,>=6.4.5->voila) (0.1.2)\n", "Requirement already satisfied: fastjsonschema in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from nbformat>=5.2.0->jupyter-server<2.0.0,>=0.3.0->voila) (2.15.1)\n", "Requirement already satisfied: six>=1.5 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from python-dateutil>=2.8.2->jupyter-client<8,>=6.1.3->voila) (1.16.0)\n", "Requirement already satisfied: argon2-cffi-bindings in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from argon2-cffi->jupyter-server<2.0.0,>=0.3.0->voila) (21.2.0)\n", "Requirement already satisfied: cffi>=1.0.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server<2.0.0,>=0.3.0->voila) (1.15.0)\n", "Requirement already satisfied: pycparser in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server<2.0.0,>=0.3.0->voila) (2.21)\n", "Requirement already satisfied: pytz>=2015.7 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from babel->jupyterlab-server<3,>=2.3.0->voila) (2021.3)\n", "Requirement already satisfied: soupsieve>1.2 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from beautifulsoup4->nbconvert<7,>=6.4.5->voila) (2.3.1)\n", "Requirement already satisfied: webencodings in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from bleach->nbconvert<7,>=6.4.5->voila) (0.5.1)\n", "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from packaging->jupyter-server<2.0.0,>=0.3.0->voila) (3.0.4)\n", "Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from requests->jupyterlab-server<3,>=2.3.0->voila) (2022.5.18.1)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from requests->jupyterlab-server<3,>=2.3.0->voila) (1.26.9)\n", "Requirement already satisfied: charset-normalizer~=2.0.0 in d:\\anaconda\\envs\\fastai2\\lib\\site-packages (from requests->jupyterlab-server<3,>=2.3.0->voila) (2.0.4)\n", "Building wheels for collected packages: json5\n", " Building wheel for json5 (setup.py): started\n", " Building wheel for json5 (setup.py): finished with status 'done'\n", " Created wheel for json5: filename=json5-0.9.8-py2.py3-none-any.whl size=18604 sha256=804f41b475133c2d56a1ebf92d1e8b8543c6e89faab0253bdc97cd410a5ff584\n", " Stored in directory: c:\\users\\michaelsnelson\\appdata\\local\\pip\\cache\\wheels\\71\\1a\\c7\\4fdc7a3d57e8dfa411ad739c67e72c1c8007c0acc95d5b030b\n", "Successfully built json5\n", "Installing collected packages: tinycss2, sniffio, websocket-client, nbconvert, anyio, jupyter-server, json5, babel, websockets, jupyterlab-server, voila\n", " Attempting uninstall: nbconvert\n", " Found existing installation: nbconvert 6.4.4\n", " Uninstalling nbconvert-6.4.4:\n", " Successfully uninstalled nbconvert-6.4.4\n", "Successfully installed anyio-3.6.1 babel-2.10.3 json5-0.9.8 jupyter-server-1.17.1 jupyterlab-server-2.14.0 nbconvert-6.5.0 sniffio-1.2.0 tinycss2-1.1.1 voila-0.3.5 websocket-client-1.3.3 websockets-10.3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Enabling: voila\n", "- Writing config: D:\\Anaconda\\envs\\fastai2\\etc\\jupyter\n", " - Validating...\n", " voila 0.3.5 ok\n" ] } ], "source": [ "\n", "!pip install voila\n", "!jupyter serverextension enable --sys-prefix voila " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Now that we have everything working in this Jupyter notebook, we can create our application. To do this, start a new notebook and add to it only the code needed to create and show the widgets that you need, and markdown for any text that you want to appear. Have a look at the *bear_classifier* notebook in the book's repo to see the simple notebook application we created.\n", "\n", "Next, install Voilà if you haven't already, by copying these lines into a notebook cell and executing it:\n", "\n", " !pip install voila\n", " !jupyter serverextension enable --sys-prefix voila\n", "\n", "Cells that begin with a `!` do not contain Python code, but instead contain code that is passed to your shell (bash, Windows PowerShell, etc.). If you are comfortable using the command line, which we'll discuss more later in this book, you can of course simply type these two lines (without the `!` prefix) directly into your terminal. In this case, the first line installs the `voila` library and application, and the second connects it to your existing Jupyter notebook.\n", "\n", "Voilà runs Jupyter notebooks just like the Jupyter notebook server you are using now does, but it also does something very important: it removes all of the cell inputs, and only shows output (including ipywidgets), along with your markdown cells. So what's left is a web application! To view your notebook as a Voilà web application, replace the word \"notebooks\" in your browser's URL with: \"voila/render\". You will see the same content as your notebook, but without any of the code cells.\n", "\n", "Of course, you don't need to use Voilà or ipywidgets. Your model is just a function you can call (`pred,pred_idx,probs = learn.predict(img)`), so you can use it with any framework, hosted on any platform. And you can take something you've prototyped in ipywidgets and Voilà and later convert it into a regular web application. We're showing you this approach in the book because we think it's a great way for data scientists and other folks that aren't web development experts to create applications from their models.\n", "\n", "We have our app, now let's deploy it!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Deploying your app" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As you now know, you need a GPU to train nearly any useful deep learning model. So, do you need a GPU to use that model in production? No! You almost certainly *do not need a GPU to serve your model in production*. There are a few reasons for this:\n", "\n", "- As we've seen, GPUs are only useful when they do lots of identical work in parallel. If you're doing (say) image classification, then you'll normally be classifying just one user's image at a time, and there isn't normally enough work to do in a single image to keep a GPU busy for long enough for it to be very efficient. So, a CPU will often be more cost-effective.\n", "- An alternative could be to wait for a few users to submit their images, and then batch them up and process them all at once on a GPU. But then you're asking your users to wait, rather than getting answers straight away! And you need a high-volume site for this to be workable. If you do need this functionality, you can use a tool such as Microsoft's [ONNX Runtime](https://github.com/microsoft/onnxruntime), or [AWS Sagemaker](https://aws.amazon.com/sagemaker/)\n", "- The complexities of dealing with GPU inference are significant. In particular, the GPU's memory will need careful manual management, and you'll need a careful queueing system to ensure you only process one batch at a time.\n", "- There's a lot more market competition in CPU than GPU servers, as a result of which there are much cheaper options available for CPU servers.\n", "\n", "Because of the complexity of GPU serving, many systems have sprung up to try to automate this. However, managing and running these systems is also complex, and generally requires compiling your model into a different form that's specialized for that system. It's typically preferable to avoid dealing with this complexity until/unless your app gets popular enough that it makes clear financial sense for you to do so." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "For at least the initial prototype of your application, and for any hobby projects that you want to show off, you can easily host them for free. The best place and the best way to do this will vary over time, so check the [book's website](https://book.fast.ai/) for the most up-to-date recommendations. As we're writing this book in early 2020 the simplest (and free!) approach is to use [Binder](https://mybinder.org/). To publish your web app on Binder, you follow these steps:\n", "\n", "1. Add your notebook to a [GitHub repository](http://github.com/).\n", "2. Paste the URL of that repo into Binder's URL, as shown in <>.\n", "3. Change the File dropdown to instead select URL.\n", "4. In the \"URL to open\" field, enter `/voila/render/name.ipynb` (replacing `name` with the name of for your notebook).\n", "5. Click the clickboard button at the bottom right to copy the URL and paste it somewhere safe. \n", "6. Click Launch." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\"Deploying" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "The first time you do this, Binder will take around 5 minutes to build your site. Behind the scenes, it is finding a virtual machine that can run your app, allocating storage, collecting the files needed for Jupyter, for your notebook, and for presenting your notebook as a web application.\n", "\n", "Finally, once it has started the app running, it will navigate your browser to your new web app. You can share the URL you copied to allow others to access your app as well.\n", "\n", "For other (both free and paid) options for deploying your web app, be sure to take a look at the [book's website](https://book.fast.ai/)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "You may well want to deploy your application onto mobile devices, or edge devices such as a Raspberry Pi. There are a lot of libraries and frameworks that allow you to integrate a model directly into a mobile application. However, these approaches tend to require a lot of extra steps and boilerplate, and do not always support all the PyTorch and fastai layers that your model might use. In addition, the work you do will depend on what kind of mobile devices you are targeting for deployment—you might need to do some work to run on iOS devices, different work to run on newer Android devices, different work for older Android devices, etc. Instead, we recommend wherever possible that you deploy the model itself to a server, and have your mobile or edge application connect to it as a web service.\n", "\n", "There are quite a few upsides to this approach. The initial installation is easier, because you only have to deploy a small GUI application, which connects to the server to do all the heavy lifting. More importantly perhaps, upgrades of that core logic can happen on your server, rather than needing to be distributed to all of your users. Your server will have a lot more memory and processing capacity than most edge devices, and it is far easier to scale those resources if your model becomes more demanding. The hardware that you will have on a server is also going to be more standard and more easily supported by fastai and PyTorch, so you don't have to compile your model into a different form.\n", "\n", "There are downsides too, of course. Your application will require a network connection, and there will be some latency each time the model is called. (It takes a while for a neural network model to run anyway, so this additional network latency may not make a big difference to your users in practice. In fact, since you can use better hardware on the server, the overall latency may even be less than if it were running locally!) Also, if your application uses sensitive data then your users may be concerned about an approach which sends that data to a remote server, so sometimes privacy considerations will mean that you need to run the model on the edge device (it may be possible to avoid this by having an *on-premise* server, such as inside a company's firewall). Managing the complexity and scaling the server can create additional overhead too, whereas if your model runs on the edge devices then each user is bringing their own compute resources, which leads to easier scaling with an increasing number of users (also known as *horizontal scaling*)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "> A: I've had a chance to see up close how the mobile ML landscape is changing in my work. We offer an iPhone app that depends on computer vision, and for years we ran our own computer vision models in the cloud. This was the only way to do it then since those models needed significant memory and compute resources and took minutes to process inputs. This approach required building not only the models (fun!) but also the infrastructure to ensure a certain number of \"compute worker machines\" were absolutely always running (scary), that more machines would automatically come online if traffic increased, that there was stable storage for large inputs and outputs, that the iOS app could know and tell the user how their job was doing, etc. Nowadays Apple provides APIs for converting models to run efficiently on device and most iOS devices have dedicated ML hardware, so that's the strategy we use for our newer models. It's still not easy but in our case it's worth it, for a faster user experience and to worry less about servers. What works for you will depend, realistically, on the user experience you're trying to create and what you personally find is easy to do. If you really know how to run servers, do it. If you really know how to build native mobile apps, do that. There are many roads up the hill.\n", "\n", "Overall, we'd recommend using a simple CPU-based server approach where possible, for as long as you can get away with it. If you're lucky enough to have a very successful application, then you'll be able to justify the investment in more complex deployment approaches at that time.\n", "\n", "Congratulations, you have successfully built a deep learning model and deployed it! Now is a good time to take a pause and think about what could go wrong." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## How to Avoid Disaster" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In practice, a deep learning model will be just one piece of a much bigger system. As we discussed at the start of this chapter, a data product requires thinking about the entire end-to-end process, from conception to use in production. In this book, we can't hope to cover all the complexity of managing deployed data products, such as managing multiple versions of models, A/B testing, canarying, refreshing the data (should we just grow and grow our datasets all the time, or should we regularly remove some of the old data?), handling data labeling, monitoring all this, detecting model rot, and so forth. In this section we will give an overview of some of the most important issues to consider; for a more detailed discussion of deployment issues we refer to you to the excellent [Building Machine Learning Powered Applications](http://shop.oreilly.com/product/0636920215912.do) by Emmanuel Ameisen (O'Reilly)\n", "\n", "One of the biggest issues to consider is that understanding and testing the behavior of a deep learning model is much more difficult than with most other code you write. With normal software development you can analyze the exact steps that the software is taking, and carefully study which of these steps match the desired behavior that you are trying to create. But with a neural network the behavior emerges from the model's attempt to match the training data, rather than being exactly defined.\n", "\n", "This can result in disaster! For instance, let's say we really were rolling out a bear detection system that will be attached to video cameras around campsites in national parks, and will warn campers of incoming bears. If we used a model trained with the dataset we downloaded there would be all kinds of problems in practice, such as:\n", "\n", "- Working with video data instead of images\n", "- Handling nighttime images, which may not appear in this dataset\n", "- Dealing with low-resolution camera images\n", "- Ensuring results are returned fast enough to be useful in practice\n", "- Recognizing bears in positions that are rarely seen in photos that people post online (for example from behind, partially covered by bushes, or when a long way away from the camera)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "A big part of the issue is that the kinds of photos that people are most likely to upload to the internet are the kinds of photos that do a good job of clearly and artistically displaying their subject matter—which isn't the kind of input this system is going to be getting. So, we may need to do a lot of our own data collection and labelling to create a useful system.\n", "\n", "This is just one example of the more general problem of *out-of-domain* data. That is to say, there may be data that our model sees in production which is very different to what it saw during training. There isn't really a complete technical solution to this problem; instead, we have to be careful about our approach to rolling out the technology.\n", "\n", "There are other reasons we need to be careful too. One very common problem is *domain shift*, where the type of data that our model sees changes over time. For instance, an insurance company may use a deep learning model as part of its pricing and risk algorithm, but over time the types of customers that the company attracts, and the types of risks they represent, may change so much that the original training data is no longer relevant.\n", "\n", "Out-of-domain data and domain shift are examples of a larger problem: that you can never fully understand the entire behaviour of your neural network. They have far too many parameters to be able to analytically understand all of their possible behaviors. This is the natural downside of their best feature—their flexibility, which enables them to solve complex problems where we may not even be able to fully specify our preferred solution approaches. The good news, however, is that there are ways to mitigate these risks using a carefully thought-out process. The details of this will vary depending on the details of the problem you are solving, but we will attempt to lay out here a high-level approach, summarized in <>, which we hope will provide useful guidance." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "\"Deployment" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Where possible, the first step is to use an entirely manual process, with your deep learning model approach running in parallel but not being used directly to drive any actions. The humans involved in the manual process should look at the deep learning outputs and check whether they make sense. For instance, with our bear classifier a park ranger could have a screen displaying video feeds from all the cameras, with any possible bear sightings simply highlighted in red. The park ranger would still be expected to be just as alert as before the model was deployed; the model is simply helping to check for problems at this point.\n", "\n", "The second step is to try to limit the scope of the model, and have it carefully supervised by people. For instance, do a small geographically and time-constrained trial of the model-driven approach. Rather than rolling our bear classifier out in every national park throughout the country, we could pick a single observation post, for a one-week period, and have a park ranger check each alert before it goes out.\n", "\n", "Then, gradually increase the scope of your rollout. As you do so, ensure that you have really good reporting systems in place, to make sure that you are aware of any significant changes to the actions being taken compared to your manual process. For instance, if the number of bear alerts doubles or halves after rollout of the new system in some location, we should be very concerned. Try to think about all the ways in which your system could go wrong, and then think about what measure or report or picture could reflect that problem, and ensure that your regular reporting includes that information." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "> J: I started a company 20 years ago called _Optimal Decisions_ that used machine learning and optimization to help giant insurance companies set their pricing, impacting tens of billions of dollars of risks. We used the approaches described here to manage the potential downsides of something going wrong. Also, before we worked with our clients to put anything in production, we tried to simulate the impact by testing the end-to-end system on their previous year's data. It was always quite a nerve-wracking process, putting these new algorithms into production, but every rollout was successful." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Unforeseen Consequences and Feedback Loops" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "One of the biggest challenges in rolling out a model is that your model may change the behaviour of the system it is a part of. For instance, consider a \"predictive policing\" algorithm that predicts more crime in certain neighborhoods, causing more police officers to be sent to those neighborhoods, which can result in more crimes being recorded in those neighborhoods, and so on. In the Royal Statistical Society paper [\"To Predict and Serve?\"](https://rss.onlinelibrary.wiley.com/doi/full/10.1111/j.1740-9713.2016.00960.x), Kristian Lum and William Isaac observe that: \"predictive policing is aptly named: it is predicting future policing, not future crime.\"\n", "\n", "Part of the issue in this case is that in the presence of bias (which we'll discuss in depth in the next chapter), *feedback loops* can result in negative implications of that bias getting worse and worse. For instance, there are concerns that this is already happening in the US, where there is significant bias in arrest rates on racial grounds. [According to the ACLU](https://www.aclu.org/issues/smart-justice/sentencing-reform/war-marijuana-black-and-white), \"despite roughly equal usage rates, Blacks are 3.73 times more likely than whites to be arrested for marijuana.\" The impact of this bias, along with the rollout of predictive policing algorithms in many parts of the US, led Bärí Williams to [write in the *New York Times*](https://www.nytimes.com/2017/12/02/opinion/sunday/intelligent-policing-and-my-innocent-children.html): \"The same technology that’s the source of so much excitement in my career is being used in law enforcement in ways that could mean that in the coming years, my son, who is 7 now, is more likely to be profiled or arrested—or worse—for no reason other than his race and where we live.\"\n", "\n", "A helpful exercise prior to rolling out a significant machine learning system is to consider this question: \"What would happen if it went really, really well?\" In other words, what if the predictive power was extremely high, and its ability to influence behavior was extremely significant? In that case, who would be most impacted? What would the most extreme results potentially look like? How would you know what was really going on?\n", "\n", "Such a thought exercise might help you to construct a more careful rollout plan, with ongoing monitoring systems and human oversight. Of course, human oversight isn't useful if it isn't listened to, so make sure that there are reliable and resilient communication channels so that the right people will be aware of issues, and will have the power to fix them." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get Writing!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "One of the things our students have found most helpful to solidify their understanding of this material is to write it down. There is no better test of your understanding of a topic than attempting to teach it to somebody else. This is helpful even if you never show your writing to anybody—but it's even better if you share it! So we recommend that, if you haven't already, you start a blog. Now that you've completed Chapter 2 and have learned how to train and deploy models, you're well placed to write your first blog post about your deep learning journey. What's surprised you? What opportunities do you see for deep learning in your field? What obstacles do you see?\n", "\n", "Rachel Thomas, cofounder of fast.ai, wrote in the article [\"Why You (Yes, You) Should Blog\"](https://medium.com/@racheltho/why-you-yes-you-should-blog-7d2544ac1045):\n", "\n", "```asciidoc\n", "____\n", "The top advice I would give my younger self would be to start blogging sooner. Here are some reasons to blog:\n", "\n", "* It’s like a resume, only better. I know of a few people who have had blog posts lead to job offers!\n", "* Helps you learn. Organizing knowledge always helps me synthesize my own ideas. One of the tests of whether you understand something is whether you can explain it to someone else. A blog post is a great way to do that.\n", "* I’ve gotten invitations to conferences and invitations to speak from my blog posts. I was invited to the TensorFlow Dev Summit (which was awesome!) for writing a blog post about how I don’t like TensorFlow.\n", "* Meet new people. I’ve met several people who have responded to blog posts I wrote.\n", "* Saves time. Any time you answer a question multiple times through email, you should turn it into a blog post, which makes it easier for you to share the next time someone asks.\n", "____\n", "```\n", "\n", "Perhaps her most important tip is this: \n", "\n", "> : You are best positioned to help people one step behind you. The material is still fresh in your mind. Many experts have forgotten what it was like to be a beginner (or an intermediate) and have forgotten why the topic is hard to understand when you first hear it. The context of your particular background, your particular style, and your knowledge level will give a different twist to what you’re writing about.\n", "\n", "We've provided full details on how to set up a blog in <>. If you don't have a blog already, take a look at that now, because we've got a really great approach set up for you to start blogging for free, with no ads—and you can even use Jupyter Notebook!" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Questionnaire" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Provide an example of where the bear classification model might work poorly in production, due to structural or style differences in the training data.\n", "1. Where do text models currently have a major deficiency?\n", "1. What are possible negative societal implications of text generation models?\n", "1. In situations where a model might make mistakes, and those mistakes could be harmful, what is a good alternative to automating a process?\n", "1. What kind of tabular data is deep learning particularly good at?\n", "1. What's a key downside of directly using a deep learning model for recommendation systems?\n", "1. What are the steps of the Drivetrain Approach?\n", "1. How do the steps of the Drivetrain Approach map to a recommendation system?\n", "1. Create an image recognition model using data you curate, and deploy it on the web.\n", "1. What is `DataLoaders`?\n", "1. What four things do we need to tell fastai to create `DataLoaders`?\n", "1. What does the `splitter` parameter to `DataBlock` do?\n", "1. How do we ensure a random split always gives the same validation set?\n", "1. What letters are often used to signify the independent and dependent variables?\n", "1. What's the difference between the crop, pad, and squish resize approaches? When might you choose one over the others?\n", "1. What is data augmentation? Why is it needed?\n", "1. What is the difference between `item_tfms` and `batch_tfms`?\n", "1. What is a confusion matrix?\n", "1. What does `export` save?\n", "1. What is it called when we use a model for getting predictions, instead of training?\n", "1. What are IPython widgets?\n", "1. When might you want to use CPU for deployment? When might GPU be better?\n", "1. What are the downsides of deploying your app to a server, instead of to a client (or edge) device such as a phone or PC?\n", "1. What are three examples of problems that could occur when rolling out a bear warning system in practice?\n", "1. What is \"out-of-domain data\"?\n", "1. What is \"domain shift\"?\n", "1. What are the three steps in the deployment process?" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Further Research" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "1. Consider how the Drivetrain Approach maps to a project or problem you're interested in.\n", "1. When might it be best to avoid certain types of data augmentation?\n", "1. For a project you're interested in applying deep learning to, consider the thought experiment \"What would happen if it went really, really well?\"\n", "1. Start a blog, and write your first blog post. For instance, write about what you think deep learning might be useful for in a domain you're interested in." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "jupytext": { "split_at_heading": true }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.4" } }, "nbformat": 4, "nbformat_minor": 4 }