{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns;\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.manifold import TSNE\n",
    "from pycaret.anomaly import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df=pd.read_csv('/mnt/c/Users/selin_uzturk/Desktop/sinkaf/encoded.csv')\n",
    "embeding_df=embeding_df.drop(['Unnamed: 0'], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8  \\\n",
       "0      101   10110     175  78653     189   25285   15976  40840    276   \n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  56  57  58  59  60  61  62  63  labels  \\\n",
       "0      31623  ...   0   0   0   0   0   0   0   0       0   \n",
       "1      23763  ...   0   0   0   0   0   0   0   0       0   \n",
       "2      76686  ...   0   0   0   0   0   0   0   0       0   \n",
       "3      10147  ...   0   0   0   0   0   0   0   0       0   \n",
       "4      13878  ...   0   0   0   0   0   0   0   0       0   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..  ..     ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0   0       1   \n",
       "43345    276  ...   0   0   0   0   0   0   0   0       1   \n",
       "43346  10561  ...   0   0   0   0   0   0   0   0       1   \n",
       "43347  26083  ...   0   0   0   0   0   0   0   0       1   \n",
       "43348  39125  ...   0   0   0   0   0   0   0   0       1   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[43349 rows x 66 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_46da2_row5_col1 {\n",
       "  background-color: lightgreen;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_46da2\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_46da2_level0_col0\" class=\"col_heading level0 col0\" >Description</th>\n",
       "      <th id=\"T_46da2_level0_col1\" class=\"col_heading level0 col1\" >Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_46da2_row0_col0\" class=\"data row0 col0\" >Session id</td>\n",
       "      <td id=\"T_46da2_row0_col1\" class=\"data row0 col1\" >5272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_46da2_row1_col0\" class=\"data row1 col0\" >Original data shape</td>\n",
       "      <td id=\"T_46da2_row1_col1\" class=\"data row1 col1\" >(43349, 66)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_46da2_row2_col0\" class=\"data row2 col0\" >Transformed data shape</td>\n",
       "      <td id=\"T_46da2_row2_col1\" class=\"data row2 col1\" >(43349, 65)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_46da2_row3_col0\" class=\"data row3 col0\" >Ignore features</td>\n",
       "      <td id=\"T_46da2_row3_col1\" class=\"data row3 col1\" >1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_46da2_row4_col0\" class=\"data row4 col0\" >Numeric features</td>\n",
       "      <td id=\"T_46da2_row4_col1\" class=\"data row4 col1\" >65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row5\" class=\"row_heading level0 row5\" >5</th>\n",
       "      <td id=\"T_46da2_row5_col0\" class=\"data row5 col0\" >Preprocess</td>\n",
       "      <td id=\"T_46da2_row5_col1\" class=\"data row5 col1\" >True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row6\" class=\"row_heading level0 row6\" >6</th>\n",
       "      <td id=\"T_46da2_row6_col0\" class=\"data row6 col0\" >Imputation type</td>\n",
       "      <td id=\"T_46da2_row6_col1\" class=\"data row6 col1\" >simple</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row7\" class=\"row_heading level0 row7\" >7</th>\n",
       "      <td id=\"T_46da2_row7_col0\" class=\"data row7 col0\" >Numeric imputation</td>\n",
       "      <td id=\"T_46da2_row7_col1\" class=\"data row7 col1\" >mean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row8\" class=\"row_heading level0 row8\" >8</th>\n",
       "      <td id=\"T_46da2_row8_col0\" class=\"data row8 col0\" >Categorical imputation</td>\n",
       "      <td id=\"T_46da2_row8_col1\" class=\"data row8 col1\" >mode</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row9\" class=\"row_heading level0 row9\" >9</th>\n",
       "      <td id=\"T_46da2_row9_col0\" class=\"data row9 col0\" >CPU Jobs</td>\n",
       "      <td id=\"T_46da2_row9_col1\" class=\"data row9 col1\" >-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row10\" class=\"row_heading level0 row10\" >10</th>\n",
       "      <td id=\"T_46da2_row10_col0\" class=\"data row10 col0\" >Use GPU</td>\n",
       "      <td id=\"T_46da2_row10_col1\" class=\"data row10 col1\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row11\" class=\"row_heading level0 row11\" >11</th>\n",
       "      <td id=\"T_46da2_row11_col0\" class=\"data row11 col0\" >Log Experiment</td>\n",
       "      <td id=\"T_46da2_row11_col1\" class=\"data row11 col1\" >False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row12\" class=\"row_heading level0 row12\" >12</th>\n",
       "      <td id=\"T_46da2_row12_col0\" class=\"data row12 col0\" >Experiment Name</td>\n",
       "      <td id=\"T_46da2_row12_col1\" class=\"data row12 col1\" >anomaly-default-name</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_46da2_level0_row13\" class=\"row_heading level0 row13\" >13</th>\n",
       "      <td id=\"T_46da2_row13_col0\" class=\"data row13 col0\" >USI</td>\n",
       "      <td id=\"T_46da2_row13_col1\" class=\"data row13 col1\" >ca74</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f58c8e68f70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ano1= setup(embeding_df,ignore_features=['tweet'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Reference</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>abod</th>\n",
       "      <td>Angle-base Outlier Detection</td>\n",
       "      <td>pyod.models.abod.ABOD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cluster</th>\n",
       "      <td>Clustering-Based Local Outlier</td>\n",
       "      <td>pycaret.internal.patches.pyod.CBLOFForceToDouble</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cof</th>\n",
       "      <td>Connectivity-Based Local Outlier</td>\n",
       "      <td>pyod.models.cof.COF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>iforest</th>\n",
       "      <td>Isolation Forest</td>\n",
       "      <td>pyod.models.iforest.IForest</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>histogram</th>\n",
       "      <td>Histogram-based Outlier Detection</td>\n",
       "      <td>pyod.models.hbos.HBOS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>knn</th>\n",
       "      <td>K-Nearest Neighbors Detector</td>\n",
       "      <td>pyod.models.knn.KNN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lof</th>\n",
       "      <td>Local Outlier Factor</td>\n",
       "      <td>pyod.models.lof.LOF</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>svm</th>\n",
       "      <td>One-class SVM detector</td>\n",
       "      <td>pyod.models.ocsvm.OCSVM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pca</th>\n",
       "      <td>Principal Component Analysis</td>\n",
       "      <td>pyod.models.pca.PCA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcd</th>\n",
       "      <td>Minimum Covariance Determinant</td>\n",
       "      <td>pyod.models.mcd.MCD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sod</th>\n",
       "      <td>Subspace Outlier Detection</td>\n",
       "      <td>pyod.models.sod.SOD</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sos</th>\n",
       "      <td>Stochastic Outlier Selection</td>\n",
       "      <td>pyod.models.sos.SOS</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Name  \\\n",
       "ID                                             \n",
       "abod            Angle-base Outlier Detection   \n",
       "cluster       Clustering-Based Local Outlier   \n",
       "cof         Connectivity-Based Local Outlier   \n",
       "iforest                     Isolation Forest   \n",
       "histogram  Histogram-based Outlier Detection   \n",
       "knn             K-Nearest Neighbors Detector   \n",
       "lof                     Local Outlier Factor   \n",
       "svm                   One-class SVM detector   \n",
       "pca             Principal Component Analysis   \n",
       "mcd           Minimum Covariance Determinant   \n",
       "sod               Subspace Outlier Detection   \n",
       "sos             Stochastic Outlier Selection   \n",
       "\n",
       "                                                  Reference  \n",
       "ID                                                           \n",
       "abod                                  pyod.models.abod.ABOD  \n",
       "cluster    pycaret.internal.patches.pyod.CBLOFForceToDouble  \n",
       "cof                                     pyod.models.cof.COF  \n",
       "iforest                         pyod.models.iforest.IForest  \n",
       "histogram                             pyod.models.hbos.HBOS  \n",
       "knn                                     pyod.models.knn.KNN  \n",
       "lof                                     pyod.models.lof.LOF  \n",
       "svm                                 pyod.models.ocsvm.OCSVM  \n",
       "pca                                     pyod.models.pca.PCA  \n",
       "mcd                                     pyod.models.mcd.MCD  \n",
       "sod                                     pyod.models.sod.SOD  \n",
       "sos                                     pyod.models.sos.SOS  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# iforest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iforest = create_model('iforest')\n",
    "# iforest_anomalies = assign_model(iforest)\n",
    "# # iso_df=embeding_df.drop(['tweet'], axis=1)\n",
    "# iforest_pred = predict_model(iforest, data=iso_df)\n",
    "# iforest_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iforest_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# knn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>Anomaly</th>\n",
       "      <th>Anomaly_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10110.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>78653.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>25285.0</td>\n",
       "      <td>15976.0</td>\n",
       "      <td>40840.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>31623.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>70971.171936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.0</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>10706.0</td>\n",
       "      <td>10713.0</td>\n",
       "      <td>10794.0</td>\n",
       "      <td>94698.0</td>\n",
       "      <td>30668.0</td>\n",
       "      <td>24883.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>23763.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>77147.550363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>30471.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>13785.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>33642.0</td>\n",
       "      <td>14399.0</td>\n",
       "      <td>48271.0</td>\n",
       "      <td>76686.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>118676.465801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.0</td>\n",
       "      <td>19319.0</td>\n",
       "      <td>16724.0</td>\n",
       "      <td>10118.0</td>\n",
       "      <td>10107.0</td>\n",
       "      <td>78323.0</td>\n",
       "      <td>12407.0</td>\n",
       "      <td>38959.0</td>\n",
       "      <td>22934.0</td>\n",
       "      <td>10147.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>94310.765409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101.0</td>\n",
       "      <td>30932.0</td>\n",
       "      <td>58706.0</td>\n",
       "      <td>58054.0</td>\n",
       "      <td>44907.0</td>\n",
       "      <td>10224.0</td>\n",
       "      <td>106583.0</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>12524.0</td>\n",
       "      <td>13878.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>63569.489655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101.0</td>\n",
       "      <td>20065.0</td>\n",
       "      <td>10161.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>103784.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>21388.0</td>\n",
       "      <td>10245.0</td>\n",
       "      <td>92067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>183310.474995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>80839.0</td>\n",
       "      <td>24109.0</td>\n",
       "      <td>13406.0</td>\n",
       "      <td>18985.0</td>\n",
       "      <td>16285.0</td>\n",
       "      <td>10163.0</td>\n",
       "      <td>11062.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>140717.435036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101.0</td>\n",
       "      <td>105549.0</td>\n",
       "      <td>102635.0</td>\n",
       "      <td>10140.0</td>\n",
       "      <td>26943.0</td>\n",
       "      <td>11499.0</td>\n",
       "      <td>110516.0</td>\n",
       "      <td>21899.0</td>\n",
       "      <td>11861.0</td>\n",
       "      <td>10561.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>98954.428628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101.0</td>\n",
       "      <td>81424.0</td>\n",
       "      <td>26398.0</td>\n",
       "      <td>92017.0</td>\n",
       "      <td>109620.0</td>\n",
       "      <td>10941.0</td>\n",
       "      <td>76010.0</td>\n",
       "      <td>10115.0</td>\n",
       "      <td>19830.0</td>\n",
       "      <td>26083.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>65424.117159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101.0</td>\n",
       "      <td>39774.0</td>\n",
       "      <td>11127.0</td>\n",
       "      <td>45989.0</td>\n",
       "      <td>24596.0</td>\n",
       "      <td>11933.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>17145.0</td>\n",
       "      <td>10710.0</td>\n",
       "      <td>39125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>182332.274049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2        3         4         5         6  \\\n",
       "0      101.0   10110.0     175.0  78653.0     189.0   25285.0   15976.0   \n",
       "1      101.0   11589.0   10706.0  10713.0   10794.0   94698.0   30668.0   \n",
       "2      101.0     148.0   30471.0  10774.0   13785.0   13779.0   33642.0   \n",
       "3      101.0   19319.0   16724.0  10118.0   10107.0   78323.0   12407.0   \n",
       "4      101.0   30932.0   58706.0  58054.0   44907.0   10224.0  106583.0   \n",
       "...      ...       ...       ...      ...       ...       ...       ...   \n",
       "43344  101.0   20065.0   10161.0    115.0     115.0  103784.0   10774.0   \n",
       "43345  101.0     139.0   80839.0  24109.0   13406.0   18985.0   16285.0   \n",
       "43346  101.0  105549.0  102635.0  10140.0   26943.0   11499.0  110516.0   \n",
       "43347  101.0   81424.0   26398.0  92017.0  109620.0   10941.0   76010.0   \n",
       "43348  101.0   39774.0   11127.0  45989.0   24596.0   11933.0     170.0   \n",
       "\n",
       "             7        8        9  ...   57   58   59   60   61   62   63  \\\n",
       "0      40840.0    276.0  31623.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1      24883.0    117.0  23763.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2      14399.0  48271.0  76686.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3      38959.0  22934.0  10147.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4      10288.0  12524.0  13878.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "...        ...      ...      ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "43344  21388.0  10245.0  92067.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43345  10163.0  11062.0    276.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43346  21899.0  11861.0  10561.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43347  10115.0  19830.0  26083.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43348  17145.0  10710.0  39125.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "       labels  Anomaly  Anomaly_Score  \n",
       "0         0.0        0   70971.171936  \n",
       "1         0.0        0   77147.550363  \n",
       "2         0.0        0  118676.465801  \n",
       "3         0.0        0   94310.765409  \n",
       "4         0.0        0   63569.489655  \n",
       "...       ...      ...            ...  \n",
       "43344     1.0        0  183310.474995  \n",
       "43345     1.0        0  140717.435036  \n",
       "43346     1.0        0   98954.428628  \n",
       "43347     1.0        0   65424.117159  \n",
       "43348     1.0        0  182332.274049  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = create_model('knn')\n",
    "knn_anomalies = assign_model(knn)\n",
    "knn_df=embeding_df.drop(['tweet'], axis=1)\n",
    "knn_pred = predict_model(knn, data=knn_df)\n",
    "knn_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    41376\n",
       "1     1973\n",
       "Name: Anomaly, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>Anomaly</th>\n",
       "      <th>Anomaly_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10110.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>78653.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>25285.0</td>\n",
       "      <td>15976.0</td>\n",
       "      <td>40840.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>31623.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>70971.171936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.0</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>10706.0</td>\n",
       "      <td>10713.0</td>\n",
       "      <td>10794.0</td>\n",
       "      <td>94698.0</td>\n",
       "      <td>30668.0</td>\n",
       "      <td>24883.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>23763.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>77147.550363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>30471.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>13785.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>33642.0</td>\n",
       "      <td>14399.0</td>\n",
       "      <td>48271.0</td>\n",
       "      <td>76686.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>118676.465801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.0</td>\n",
       "      <td>19319.0</td>\n",
       "      <td>16724.0</td>\n",
       "      <td>10118.0</td>\n",
       "      <td>10107.0</td>\n",
       "      <td>78323.0</td>\n",
       "      <td>12407.0</td>\n",
       "      <td>38959.0</td>\n",
       "      <td>22934.0</td>\n",
       "      <td>10147.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>94310.765409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101.0</td>\n",
       "      <td>30932.0</td>\n",
       "      <td>58706.0</td>\n",
       "      <td>58054.0</td>\n",
       "      <td>44907.0</td>\n",
       "      <td>10224.0</td>\n",
       "      <td>106583.0</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>12524.0</td>\n",
       "      <td>13878.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>63569.489655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101.0</td>\n",
       "      <td>20065.0</td>\n",
       "      <td>10161.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>103784.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>21388.0</td>\n",
       "      <td>10245.0</td>\n",
       "      <td>92067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>183310.474995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>80839.0</td>\n",
       "      <td>24109.0</td>\n",
       "      <td>13406.0</td>\n",
       "      <td>18985.0</td>\n",
       "      <td>16285.0</td>\n",
       "      <td>10163.0</td>\n",
       "      <td>11062.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>140717.435036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101.0</td>\n",
       "      <td>105549.0</td>\n",
       "      <td>102635.0</td>\n",
       "      <td>10140.0</td>\n",
       "      <td>26943.0</td>\n",
       "      <td>11499.0</td>\n",
       "      <td>110516.0</td>\n",
       "      <td>21899.0</td>\n",
       "      <td>11861.0</td>\n",
       "      <td>10561.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>98954.428628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101.0</td>\n",
       "      <td>81424.0</td>\n",
       "      <td>26398.0</td>\n",
       "      <td>92017.0</td>\n",
       "      <td>109620.0</td>\n",
       "      <td>10941.0</td>\n",
       "      <td>76010.0</td>\n",
       "      <td>10115.0</td>\n",
       "      <td>19830.0</td>\n",
       "      <td>26083.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>65424.117159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101.0</td>\n",
       "      <td>39774.0</td>\n",
       "      <td>11127.0</td>\n",
       "      <td>45989.0</td>\n",
       "      <td>24596.0</td>\n",
       "      <td>11933.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>17145.0</td>\n",
       "      <td>10710.0</td>\n",
       "      <td>39125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>182332.274049</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2        3         4         5         6  \\\n",
       "0      101.0   10110.0     175.0  78653.0     189.0   25285.0   15976.0   \n",
       "1      101.0   11589.0   10706.0  10713.0   10794.0   94698.0   30668.0   \n",
       "2      101.0     148.0   30471.0  10774.0   13785.0   13779.0   33642.0   \n",
       "3      101.0   19319.0   16724.0  10118.0   10107.0   78323.0   12407.0   \n",
       "4      101.0   30932.0   58706.0  58054.0   44907.0   10224.0  106583.0   \n",
       "...      ...       ...       ...      ...       ...       ...       ...   \n",
       "43344  101.0   20065.0   10161.0    115.0     115.0  103784.0   10774.0   \n",
       "43345  101.0     139.0   80839.0  24109.0   13406.0   18985.0   16285.0   \n",
       "43346  101.0  105549.0  102635.0  10140.0   26943.0   11499.0  110516.0   \n",
       "43347  101.0   81424.0   26398.0  92017.0  109620.0   10941.0   76010.0   \n",
       "43348  101.0   39774.0   11127.0  45989.0   24596.0   11933.0     170.0   \n",
       "\n",
       "             7        8        9  ...   57   58   59   60   61   62   63  \\\n",
       "0      40840.0    276.0  31623.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1      24883.0    117.0  23763.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2      14399.0  48271.0  76686.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3      38959.0  22934.0  10147.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4      10288.0  12524.0  13878.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "...        ...      ...      ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "43344  21388.0  10245.0  92067.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43345  10163.0  11062.0    276.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43346  21899.0  11861.0  10561.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43347  10115.0  19830.0  26083.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43348  17145.0  10710.0  39125.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "       labels  Anomaly  Anomaly_Score  \n",
       "0         0.0        0   70971.171936  \n",
       "1         0.0        0   77147.550363  \n",
       "2         0.0        0  118676.465801  \n",
       "3         0.0        0   94310.765409  \n",
       "4         0.0        0   63569.489655  \n",
       "...       ...      ...            ...  \n",
       "43344     1.0        0  183310.474995  \n",
       "43345     1.0        0  140717.435036  \n",
       "43346     1.0        0   98954.428628  \n",
       "43347     1.0        0   65424.117159  \n",
       "43348     1.0        0  182332.274049  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    41376\n",
       "1     1973\n",
       "Name: Anomaly, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8  \\\n",
       "0      101   10110     175  78653     189   25285   15976  40840    276   \n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  56  57  58  59  60  61  62  63  labels  \\\n",
       "0      31623  ...   0   0   0   0   0   0   0   0       0   \n",
       "1      23763  ...   0   0   0   0   0   0   0   0       0   \n",
       "2      76686  ...   0   0   0   0   0   0   0   0       0   \n",
       "3      10147  ...   0   0   0   0   0   0   0   0       0   \n",
       "4      13878  ...   0   0   0   0   0   0   0   0       0   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..  ..     ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0   0       1   \n",
       "43345    276  ...   0   0   0   0   0   0   0   0       1   \n",
       "43346  10561  ...   0   0   0   0   0   0   0   0       1   \n",
       "43347  26083  ...   0   0   0   0   0   0   0   0       1   \n",
       "43348  39125  ...   0   0   0   0   0   0   0   0       1   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[43349 rows x 66 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df.drop(knn_pred.loc[knn_pred['Anomaly']==1 ].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41376 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8  \\\n",
       "0      101   10110     175  78653     189   25285   15976  40840    276   \n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  56  57  58  59  60  61  62  63  labels  \\\n",
       "0      31623  ...   0   0   0   0   0   0   0   0       0   \n",
       "1      23763  ...   0   0   0   0   0   0   0   0       0   \n",
       "2      76686  ...   0   0   0   0   0   0   0   0       0   \n",
       "3      10147  ...   0   0   0   0   0   0   0   0       0   \n",
       "4      13878  ...   0   0   0   0   0   0   0   0       0   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..  ..     ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0   0       1   \n",
       "43345    276  ...   0   0   0   0   0   0   0   0       1   \n",
       "43346  10561  ...   0   0   0   0   0   0   0   0       1   \n",
       "43347  26083  ...   0   0   0   0   0   0   0   0       1   \n",
       "43348  39125  ...   0   0   0   0   0   0   0   0       1   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[41376 rows x 66 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame()\n",
    "df['tweet']=embeding_df['tweet']\n",
    "df['subtas_a']=embeding_df['labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('knn_outliers.csv')  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# pca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>Anomaly</th>\n",
       "      <th>Anomaly_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10110.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>78653.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>25285.0</td>\n",
       "      <td>15976.0</td>\n",
       "      <td>40840.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>31623.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.354399e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.0</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>10706.0</td>\n",
       "      <td>10713.0</td>\n",
       "      <td>10794.0</td>\n",
       "      <td>94698.0</td>\n",
       "      <td>30668.0</td>\n",
       "      <td>24883.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>23763.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.311723e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>30471.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>13785.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>33642.0</td>\n",
       "      <td>14399.0</td>\n",
       "      <td>48271.0</td>\n",
       "      <td>76686.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.597792e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.0</td>\n",
       "      <td>19319.0</td>\n",
       "      <td>16724.0</td>\n",
       "      <td>10118.0</td>\n",
       "      <td>10107.0</td>\n",
       "      <td>78323.0</td>\n",
       "      <td>12407.0</td>\n",
       "      <td>38959.0</td>\n",
       "      <td>22934.0</td>\n",
       "      <td>10147.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.551488e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101.0</td>\n",
       "      <td>30932.0</td>\n",
       "      <td>58706.0</td>\n",
       "      <td>58054.0</td>\n",
       "      <td>44907.0</td>\n",
       "      <td>10224.0</td>\n",
       "      <td>106583.0</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>12524.0</td>\n",
       "      <td>13878.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.348867e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101.0</td>\n",
       "      <td>20065.0</td>\n",
       "      <td>10161.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>103784.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>21388.0</td>\n",
       "      <td>10245.0</td>\n",
       "      <td>92067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.346619e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>80839.0</td>\n",
       "      <td>24109.0</td>\n",
       "      <td>13406.0</td>\n",
       "      <td>18985.0</td>\n",
       "      <td>16285.0</td>\n",
       "      <td>10163.0</td>\n",
       "      <td>11062.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.778253e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101.0</td>\n",
       "      <td>105549.0</td>\n",
       "      <td>102635.0</td>\n",
       "      <td>10140.0</td>\n",
       "      <td>26943.0</td>\n",
       "      <td>11499.0</td>\n",
       "      <td>110516.0</td>\n",
       "      <td>21899.0</td>\n",
       "      <td>11861.0</td>\n",
       "      <td>10561.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.762300e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101.0</td>\n",
       "      <td>81424.0</td>\n",
       "      <td>26398.0</td>\n",
       "      <td>92017.0</td>\n",
       "      <td>109620.0</td>\n",
       "      <td>10941.0</td>\n",
       "      <td>76010.0</td>\n",
       "      <td>10115.0</td>\n",
       "      <td>19830.0</td>\n",
       "      <td>26083.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.564075e+32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101.0</td>\n",
       "      <td>39774.0</td>\n",
       "      <td>11127.0</td>\n",
       "      <td>45989.0</td>\n",
       "      <td>24596.0</td>\n",
       "      <td>11933.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>17145.0</td>\n",
       "      <td>10710.0</td>\n",
       "      <td>39125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.685411e+32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2        3         4         5         6  \\\n",
       "0      101.0   10110.0     175.0  78653.0     189.0   25285.0   15976.0   \n",
       "1      101.0   11589.0   10706.0  10713.0   10794.0   94698.0   30668.0   \n",
       "2      101.0     148.0   30471.0  10774.0   13785.0   13779.0   33642.0   \n",
       "3      101.0   19319.0   16724.0  10118.0   10107.0   78323.0   12407.0   \n",
       "4      101.0   30932.0   58706.0  58054.0   44907.0   10224.0  106583.0   \n",
       "...      ...       ...       ...      ...       ...       ...       ...   \n",
       "43344  101.0   20065.0   10161.0    115.0     115.0  103784.0   10774.0   \n",
       "43345  101.0     139.0   80839.0  24109.0   13406.0   18985.0   16285.0   \n",
       "43346  101.0  105549.0  102635.0  10140.0   26943.0   11499.0  110516.0   \n",
       "43347  101.0   81424.0   26398.0  92017.0  109620.0   10941.0   76010.0   \n",
       "43348  101.0   39774.0   11127.0  45989.0   24596.0   11933.0     170.0   \n",
       "\n",
       "             7        8        9  ...   57   58   59   60   61   62   63  \\\n",
       "0      40840.0    276.0  31623.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1      24883.0    117.0  23763.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2      14399.0  48271.0  76686.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3      38959.0  22934.0  10147.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4      10288.0  12524.0  13878.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "...        ...      ...      ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "43344  21388.0  10245.0  92067.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43345  10163.0  11062.0    276.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43346  21899.0  11861.0  10561.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43347  10115.0  19830.0  26083.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43348  17145.0  10710.0  39125.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "       labels  Anomaly  Anomaly_Score  \n",
       "0         0.0        0   1.354399e+32  \n",
       "1         0.0        0   1.311723e+32  \n",
       "2         0.0        0   1.597792e+32  \n",
       "3         0.0        0   1.551488e+32  \n",
       "4         0.0        0   1.348867e+32  \n",
       "...       ...      ...            ...  \n",
       "43344     1.0        0   2.346619e+32  \n",
       "43345     1.0        0   1.778253e+32  \n",
       "43346     1.0        0   1.762300e+32  \n",
       "43347     1.0        0   1.564075e+32  \n",
       "43348     1.0        0   2.685411e+32  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca = create_model('pca')\n",
    "pca_anomalies = assign_model(pca)\n",
    "pca_df=embeding_df.drop(['tweet'], axis=1)\n",
    "pca_pred = predict_model(pca, data=pca_df)\n",
    "pca_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    41181\n",
       "1     2168\n",
       "Name: Anomaly, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pca_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df.drop(pca_pred.loc[pca_pred['Anomaly']==1 ].index, inplace=True)\n",
    "df=pd.DataFrame()\n",
    "df['tweet']=embeding_df['tweet']\n",
    "df['subtas_a']=embeding_df['labels']\n",
    "df.to_csv('pca_outliers.csv')  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# abod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>Anomaly</th>\n",
       "      <th>Anomaly_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10110.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>78653.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>25285.0</td>\n",
       "      <td>15976.0</td>\n",
       "      <td>40840.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>31623.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-7.719921e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.0</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>10706.0</td>\n",
       "      <td>10713.0</td>\n",
       "      <td>10794.0</td>\n",
       "      <td>94698.0</td>\n",
       "      <td>30668.0</td>\n",
       "      <td>24883.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>23763.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.030618e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>30471.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>13785.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>33642.0</td>\n",
       "      <td>14399.0</td>\n",
       "      <td>48271.0</td>\n",
       "      <td>76686.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-3.558939e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.0</td>\n",
       "      <td>19319.0</td>\n",
       "      <td>16724.0</td>\n",
       "      <td>10118.0</td>\n",
       "      <td>10107.0</td>\n",
       "      <td>78323.0</td>\n",
       "      <td>12407.0</td>\n",
       "      <td>38959.0</td>\n",
       "      <td>22934.0</td>\n",
       "      <td>10147.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-2.895136e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101.0</td>\n",
       "      <td>30932.0</td>\n",
       "      <td>58706.0</td>\n",
       "      <td>58054.0</td>\n",
       "      <td>44907.0</td>\n",
       "      <td>10224.0</td>\n",
       "      <td>106583.0</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>12524.0</td>\n",
       "      <td>13878.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-4.832515e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101.0</td>\n",
       "      <td>20065.0</td>\n",
       "      <td>10161.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>103784.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>21388.0</td>\n",
       "      <td>10245.0</td>\n",
       "      <td>92067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-7.984637e-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>80839.0</td>\n",
       "      <td>24109.0</td>\n",
       "      <td>13406.0</td>\n",
       "      <td>18985.0</td>\n",
       "      <td>16285.0</td>\n",
       "      <td>10163.0</td>\n",
       "      <td>11062.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.059387e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101.0</td>\n",
       "      <td>105549.0</td>\n",
       "      <td>102635.0</td>\n",
       "      <td>10140.0</td>\n",
       "      <td>26943.0</td>\n",
       "      <td>11499.0</td>\n",
       "      <td>110516.0</td>\n",
       "      <td>21899.0</td>\n",
       "      <td>11861.0</td>\n",
       "      <td>10561.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-3.592603e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101.0</td>\n",
       "      <td>81424.0</td>\n",
       "      <td>26398.0</td>\n",
       "      <td>92017.0</td>\n",
       "      <td>109620.0</td>\n",
       "      <td>10941.0</td>\n",
       "      <td>76010.0</td>\n",
       "      <td>10115.0</td>\n",
       "      <td>19830.0</td>\n",
       "      <td>26083.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-2.226002e-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101.0</td>\n",
       "      <td>39774.0</td>\n",
       "      <td>11127.0</td>\n",
       "      <td>45989.0</td>\n",
       "      <td>24596.0</td>\n",
       "      <td>11933.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>17145.0</td>\n",
       "      <td>10710.0</td>\n",
       "      <td>39125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>-2.864757e-23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2        3         4         5         6  \\\n",
       "0      101.0   10110.0     175.0  78653.0     189.0   25285.0   15976.0   \n",
       "1      101.0   11589.0   10706.0  10713.0   10794.0   94698.0   30668.0   \n",
       "2      101.0     148.0   30471.0  10774.0   13785.0   13779.0   33642.0   \n",
       "3      101.0   19319.0   16724.0  10118.0   10107.0   78323.0   12407.0   \n",
       "4      101.0   30932.0   58706.0  58054.0   44907.0   10224.0  106583.0   \n",
       "...      ...       ...       ...      ...       ...       ...       ...   \n",
       "43344  101.0   20065.0   10161.0    115.0     115.0  103784.0   10774.0   \n",
       "43345  101.0     139.0   80839.0  24109.0   13406.0   18985.0   16285.0   \n",
       "43346  101.0  105549.0  102635.0  10140.0   26943.0   11499.0  110516.0   \n",
       "43347  101.0   81424.0   26398.0  92017.0  109620.0   10941.0   76010.0   \n",
       "43348  101.0   39774.0   11127.0  45989.0   24596.0   11933.0     170.0   \n",
       "\n",
       "             7        8        9  ...   57   58   59   60   61   62   63  \\\n",
       "0      40840.0    276.0  31623.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1      24883.0    117.0  23763.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2      14399.0  48271.0  76686.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3      38959.0  22934.0  10147.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4      10288.0  12524.0  13878.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "...        ...      ...      ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "43344  21388.0  10245.0  92067.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43345  10163.0  11062.0    276.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43346  21899.0  11861.0  10561.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43347  10115.0  19830.0  26083.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43348  17145.0  10710.0  39125.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "       labels  Anomaly  Anomaly_Score  \n",
       "0         0.0        0  -7.719921e-22  \n",
       "1         0.0        0  -4.030618e-21  \n",
       "2         0.0        0  -3.558939e-22  \n",
       "3         0.0        0  -2.895136e-22  \n",
       "4         0.0        0  -4.832515e-21  \n",
       "...       ...      ...            ...  \n",
       "43344     1.0        0  -7.984637e-25  \n",
       "43345     1.0        0  -1.059387e-22  \n",
       "43346     1.0        0  -3.592603e-22  \n",
       "43347     1.0        0  -2.226002e-21  \n",
       "43348     1.0        0  -2.864757e-23  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abod = create_model('abod')\n",
    "abod_anomalies = assign_model(abod)\n",
    "abod_df=embeding_df.drop(['tweet'], axis=1)\n",
    "abod_pred = predict_model(abod, data=abod_df)\n",
    "abod_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    43349\n",
       "Name: Anomaly, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abod_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Initiated</th>\n",
       "      <td>. . . . . . . . . . . . . . . . . .</td>\n",
       "      <td>16:19:48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Status</th>\n",
       "      <td>. . . . . . . . . . . . . . . . . .</td>\n",
       "      <td>Fitting 0.05 Fraction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Estimator</th>\n",
       "      <td>. . . . . . . . . . . . . . . . . .</td>\n",
       "      <td>Clustering-Based Local Outlier</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                              \n",
       "                                                                              \n",
       "Initiated  . . . . . . . . . . . . . . . . . .                        16:19:48\n",
       "Status     . . . . . . . . . . . . . . . . . .           Fitting 0.05 Fraction\n",
       "Estimator  . . . . . . . . . . . . . . . . . .  Clustering-Based Local Outlier"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>Anomaly</th>\n",
       "      <th>Anomaly_Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10110.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>78653.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>25285.0</td>\n",
       "      <td>15976.0</td>\n",
       "      <td>40840.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>31623.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>123828.159076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101.0</td>\n",
       "      <td>11589.0</td>\n",
       "      <td>10706.0</td>\n",
       "      <td>10713.0</td>\n",
       "      <td>10794.0</td>\n",
       "      <td>94698.0</td>\n",
       "      <td>30668.0</td>\n",
       "      <td>24883.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>23763.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>112972.396566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101.0</td>\n",
       "      <td>148.0</td>\n",
       "      <td>30471.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>13785.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>33642.0</td>\n",
       "      <td>14399.0</td>\n",
       "      <td>48271.0</td>\n",
       "      <td>76686.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>145701.165368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.0</td>\n",
       "      <td>19319.0</td>\n",
       "      <td>16724.0</td>\n",
       "      <td>10118.0</td>\n",
       "      <td>10107.0</td>\n",
       "      <td>78323.0</td>\n",
       "      <td>12407.0</td>\n",
       "      <td>38959.0</td>\n",
       "      <td>22934.0</td>\n",
       "      <td>10147.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>141686.216880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101.0</td>\n",
       "      <td>30932.0</td>\n",
       "      <td>58706.0</td>\n",
       "      <td>58054.0</td>\n",
       "      <td>44907.0</td>\n",
       "      <td>10224.0</td>\n",
       "      <td>106583.0</td>\n",
       "      <td>10288.0</td>\n",
       "      <td>12524.0</td>\n",
       "      <td>13878.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>101399.757887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101.0</td>\n",
       "      <td>20065.0</td>\n",
       "      <td>10161.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>103784.0</td>\n",
       "      <td>10774.0</td>\n",
       "      <td>21388.0</td>\n",
       "      <td>10245.0</td>\n",
       "      <td>92067.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>193403.127721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>80839.0</td>\n",
       "      <td>24109.0</td>\n",
       "      <td>13406.0</td>\n",
       "      <td>18985.0</td>\n",
       "      <td>16285.0</td>\n",
       "      <td>10163.0</td>\n",
       "      <td>11062.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>154821.530684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101.0</td>\n",
       "      <td>105549.0</td>\n",
       "      <td>102635.0</td>\n",
       "      <td>10140.0</td>\n",
       "      <td>26943.0</td>\n",
       "      <td>11499.0</td>\n",
       "      <td>110516.0</td>\n",
       "      <td>21899.0</td>\n",
       "      <td>11861.0</td>\n",
       "      <td>10561.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>166024.182457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101.0</td>\n",
       "      <td>81424.0</td>\n",
       "      <td>26398.0</td>\n",
       "      <td>92017.0</td>\n",
       "      <td>109620.0</td>\n",
       "      <td>10941.0</td>\n",
       "      <td>76010.0</td>\n",
       "      <td>10115.0</td>\n",
       "      <td>19830.0</td>\n",
       "      <td>26083.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>130852.856537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101.0</td>\n",
       "      <td>39774.0</td>\n",
       "      <td>11127.0</td>\n",
       "      <td>45989.0</td>\n",
       "      <td>24596.0</td>\n",
       "      <td>11933.0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>17145.0</td>\n",
       "      <td>10710.0</td>\n",
       "      <td>39125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>206109.572124</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2        3         4         5         6  \\\n",
       "0      101.0   10110.0     175.0  78653.0     189.0   25285.0   15976.0   \n",
       "1      101.0   11589.0   10706.0  10713.0   10794.0   94698.0   30668.0   \n",
       "2      101.0     148.0   30471.0  10774.0   13785.0   13779.0   33642.0   \n",
       "3      101.0   19319.0   16724.0  10118.0   10107.0   78323.0   12407.0   \n",
       "4      101.0   30932.0   58706.0  58054.0   44907.0   10224.0  106583.0   \n",
       "...      ...       ...       ...      ...       ...       ...       ...   \n",
       "43344  101.0   20065.0   10161.0    115.0     115.0  103784.0   10774.0   \n",
       "43345  101.0     139.0   80839.0  24109.0   13406.0   18985.0   16285.0   \n",
       "43346  101.0  105549.0  102635.0  10140.0   26943.0   11499.0  110516.0   \n",
       "43347  101.0   81424.0   26398.0  92017.0  109620.0   10941.0   76010.0   \n",
       "43348  101.0   39774.0   11127.0  45989.0   24596.0   11933.0     170.0   \n",
       "\n",
       "             7        8        9  ...   57   58   59   60   61   62   63  \\\n",
       "0      40840.0    276.0  31623.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "1      24883.0    117.0  23763.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "2      14399.0  48271.0  76686.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "3      38959.0  22934.0  10147.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "4      10288.0  12524.0  13878.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "...        ...      ...      ...  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "43344  21388.0  10245.0  92067.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43345  10163.0  11062.0    276.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43346  21899.0  11861.0  10561.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43347  10115.0  19830.0  26083.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "43348  17145.0  10710.0  39125.0  ...  0.0  0.0  0.0  0.0  0.0  0.0  0.0   \n",
       "\n",
       "       labels  Anomaly  Anomaly_Score  \n",
       "0         0.0        0  123828.159076  \n",
       "1         0.0        0  112972.396566  \n",
       "2         0.0        0  145701.165368  \n",
       "3         0.0        0  141686.216880  \n",
       "4         0.0        0  101399.757887  \n",
       "...       ...      ...            ...  \n",
       "43344     1.0        0  193403.127721  \n",
       "43345     1.0        0  154821.530684  \n",
       "43346     1.0        0  166024.182457  \n",
       "43347     1.0        0  130852.856537  \n",
       "43348     1.0        0  206109.572124  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster = create_model('cluster')\n",
    "cluster_anomalies = assign_model(cluster)\n",
    "cluster_df=embeding_df.drop(['tweet'], axis=1)\n",
    "cluster_pred = predict_model(cluster, data=cluster_df)\n",
    "cluster_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    41182\n",
       "1     2167\n",
       "Name: Anomaly, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df.drop(cluster_pred.loc[cluster_pred['Anomaly']==1 ].index, inplace=True)\n",
    "df=pd.DataFrame()\n",
    "df['tweet']=embeding_df['tweet']\n",
    "df['subtas_a']=embeding_df['labels']\n",
    "df.to_csv('cluster_outliers.csv')  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cof"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cof = create_model('cof')\n",
    "cof_anomalies = assign_model(cof)\n",
    "cof_df=embeding_df.drop(['tweet'], axis=1)\n",
    "cof_pred = predict_model(cof, data=cof_df)\n",
    "cof_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cof_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "histogram = create_model('histogram')\n",
    "histogram_anomalies = assign_model(histogram)\n",
    "histogram_df=embeding_df.drop(['tweet'], axis=1)\n",
    "histogram_pred = predict_model(histogram, data=histogram_df)\n",
    "histogram_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "histogram_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lof"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lof = create_model('lof')\n",
    "lof_anomalies = assign_model(lof)\n",
    "lof_df=embeding_df.drop(['tweet'], axis=1)\n",
    "lof_pred = predict_model(lof, data=lof_df)\n",
    "lof_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lof_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lof = create_model('lof')\n",
    "lof_anomalies = assign_model(lof)\n",
    "lof_df=embeding_df.drop(['tweet'], axis=1)\n",
    "lof_pred = predict_model(lof, data=lof_df)\n",
    "lof_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lof_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# mcd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcd = create_model('mcd')\n",
    "mcd_anomalies = assign_model(mcd)\n",
    "mcd_df=embeding_df.drop(['tweet'], axis=1)\n",
    "mcd_pred = predict_model(mcd, data=mcd_df)\n",
    "mcd_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcd_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sod = create_model('sod')\n",
    "sod_anomalies = assign_model(sod)\n",
    "sod_df=embeding_df.drop(['tweet'], axis=1)\n",
    "sod_pred = predict_model(sod, data=sod_df)\n",
    "sod_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sod_pred['Anomaly'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sos = create_model('sos')\n",
    "sos_anomalies = assign_model(sos)\n",
    "sos_df=embeding_df.drop(['tweet'], axis=1)\n",
    "sos_pred = predict_model(sos, data=sos_df)\n",
    "sos_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sos_pred['Anomaly'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dl_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}