Delete Building_a_Safety_Agent.ipynb

Building_a_Safety_Agent.ipynb

@@ -1,316 +0,0 @@
-{
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "bdp9fSdWKBhp",
-        "collapsed": true
-      },
-      "outputs": [],
-      "source": [
-        "# Install the Libraries used in this notebook.\n",
-        "\n",
-        "!pip install -qU langchain openai transformers selfcheckgpt profanityfilter\n",
-        "! python -m spacy download en"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "##Safety Agent"
-      ],
-      "metadata": {
-        "id": "6wj7wxo9aTe5"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.chat_models import ChatOpenAI\n",
-        "from langchain.chains.conversation.memory import ConversationBufferWindowMemory\n",
-        "import openai\n",
-        "import os\n",
-        "os.environ['OPENAI_API_KEY'] = openai.api_key= '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'  #platform.openai.com api key\n",
-        "\n",
-        "# initialize LLM\n",
-        "llm = ChatOpenAI(\n",
-        "    temperature=0,\n",
-        "    model_name='gpt-3.5-turbo'\n",
-        ")\n",
-        "\n",
-        "# initialize conversational memory\n",
-        "conversational_memory = ConversationBufferWindowMemory(\n",
-        "    memory_key='chat_history',\n",
-        "    k=5,\n",
-        "    return_messages=True\n",
-        ")"
-      ],
-      "metadata": {
-        "id": "OIKitpN-fPSF"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Profanity Detection Tool"
-      ],
-      "metadata": {
-        "id": "O6BLtKefgSxZ"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from profanityfilter import ProfanityFilter\n",
-        "import spacy\n",
-        "from langchain.tools import BaseTool\n",
-        "from typing import Optional\n",
-        "\n",
-        "class Profanity_Check(BaseTool):\n",
-        "    name = \"Profanity_Checker\"\n",
-        "    description = (\n",
-        "        \"use this tool when you need to check for profanity in given text\"\n",
-        "    )\n",
-        "    def _run(\n",
-        "      self,\n",
-        "      sentence1: Optional[str] = None\n",
-        "      ):\n",
-        "      pf = ProfanityFilter()\n",
-        "      flag = pf.is_profane(sentence1)\n",
-        "      if flag:  return 'Profanity Detected'\n",
-        "      else: return 'No Profanity found'\n",
-        "\n",
-        "\n",
-        "    def _arun(self, sentence1, sentence2):\n",
-        "      raise NotImplementedError(\"This tool does not support async runs.\")\n"
-      ],
-      "metadata": {
-        "id": "_SJa13N5i1kY"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Hallucination Detection Tool"
-      ],
-      "metadata": {
-        "id": "YicTP78lgXlT"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from selfcheckgpt.modeling_selfcheck import SelfCheckBERTScore\n",
-        "\n",
-        "class Hallucination_Scorer(BaseTool):\n",
-        "    name = \"Hallucination_Scorer\"\n",
-        "    description = (\n",
-        "        \"use this tool when a you need to give hallucination scores\"\n",
-        "    )\n",
-        "    def _run(\n",
-        "      self,\n",
-        "      sentence1: Optional[str] = None\n",
-        "      ):\n",
-        "      selfcheck_bertscore = SelfCheckBERTScore()\n",
-        "      nlp = spacy.load('en_core_web_sm')\n",
-        "      passage = sentence1\n",
-        "      sentences = [sent.text.strip() for sent in nlp(passage).sents]\n",
-        "\n",
-        "      chat_completion = openai.ChatCompletion.create(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": sentence1}])\n",
-        "      sample1 = chat_completion.choices[0].message.content\n",
-        "      chat_completion = openai.ChatCompletion.create(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": sentence1}])\n",
-        "      sample2 = chat_completion.choices[0].message.content\n",
-        "      chat_completion = openai.ChatCompletion.create(model=\"gpt-3.5-turbo\", messages=[{\"role\": \"user\", \"content\": sentence1}])\n",
-        "      sample3 = chat_completion.choices[0].message.content\n",
-        "# SelfCheck-BERTScore: Score for each sentence where value is in [0.0, 1.0] and high value means non-factual\n",
-        "      sent_scores_bertscore = selfcheck_bertscore.predict(\n",
-        "          sentences = sentences,                          # list of sentences\n",
-        "          sampled_passages = [sample1, sample2, sample3], # list of sampled passages\n",
-        "          )\n",
-        "      return sent_scores_bertscore\n",
-        "\n",
-        "\n",
-        "    def _arun(self, sentence1, sentence2):\n",
-        "      raise NotImplementedError(\"This tool does not support async runs.\")\n"
-      ],
-      "metadata": {
-        "id": "1CA0tBsWYC6K"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Initializing Safety Agent\n"
-      ],
-      "metadata": {
-        "id": "6wqgeEvKgex7"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "from langchain.agents import initialize_agent\n",
-        "\n",
-        "# Pass the tools\n",
-        "tools = [Hallucination_Scorer(),Profanity_Check()]\n",
-        "\n",
-        "# initialize agent with tools\n",
-        "agent = initialize_agent(\n",
-        "    agent='chat-conversational-react-description',\n",
-        "    tools=tools, # Point each smaller sized agent towards the test we use\n",
-        "    llm=llm, # Can be buiult over any LLM\n",
-        "    verbose=True, ## Temperature for responses is set to zero for determinsitc test score// change to 1 when generating reports.\n",
-        "    max_iterations=3,  # Avoid Looping\n",
-        "    early_stopping_method='generate', # Stop and generate a score\n",
-        "    memory=conversational_memory # Chat Memory\n",
-        "\n",
-        ")"
-      ],
-      "metadata": {
-        "id": "4hI5kL3I10sM"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "##### Loading generated policy"
-      ],
-      "metadata": {
-        "id": "RNxBUVOsjXpr"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "with open(\"/content/Generated_Policy.txt\", \"r\") as file1:\n",
-        "    text = file1.read()\n",
-        "    generated_policy = ' '.join(text.split('\\n'))\n",
-        "file1.close()"
-      ],
-      "metadata": {
-        "id": "exp-mK78gbUG",
-        "collapsed": true
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "generated_policy"
-      ],
-      "metadata": {
-        "id": "KsGtYtrkgpey"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Hallucination Scoring"
-      ],
-      "metadata": {
-        "id": "2_32mHKGjeqt"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "prompt1 = 'Hallucination score for :'+generated_policy"
-      ],
-      "metadata": {
-        "id": "VWH3VDo03oHr"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "var1 = agent(prompt1)"
-      ],
-      "metadata": {
-        "id": "8aCm3m79hppI"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "var1['output']"
-      ],
-      "metadata": {
-        "id": "gum7V3J9QFBv"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
-      "source": [
-        "### Profanity Detection"
-      ],
-      "metadata": {
-        "id": "yjyYQJj0jlxm"
-      }
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "prompt2 = 'Check for profanity in '+generated_policy\n"
-      ],
-      "metadata": {
-        "id": "xoc4nCbr4Im5"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "var2 = agent(prompt2)"
-      ],
-      "metadata": {
-        "id": "fBsWajM4lOyo"
-      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
-      "source": [
-        "var2['output']"
-      ],
-      "metadata": {
-        "id": "Nc6aYJ1eYnpf"
-      },
-      "execution_count": null,
-      "outputs": []
-    }
-  ]
-}