{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.manifold import TSNE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df=pd.read_csv('/mnt/c/Users/selin_uzturk/Desktop/sinkaf/encoded.csv')\n",
    "embeding_df=embeding_df.drop(['Unnamed: 0'], axis=1)\n",
    "copy_df=pd.read_csv('/mnt/c/Users/selin_uzturk/Desktop/sinkaf/encoded.csv')\n",
    "copy_df=copy_df.drop(['Unnamed: 0'], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8   \n",
       "0      101   10110     175  78653     189   25285   15976  40840    276  \\\n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  56  57  58  59  60  61  62  63  labels   \n",
       "0      31623  ...   0   0   0   0   0   0   0   0       0  \\\n",
       "1      23763  ...   0   0   0   0   0   0   0   0       0   \n",
       "2      76686  ...   0   0   0   0   0   0   0   0       0   \n",
       "3      10147  ...   0   0   0   0   0   0   0   0       0   \n",
       "4      13878  ...   0   0   0   0   0   0   0   0       0   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..  ..     ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0   0       1   \n",
       "43345    276  ...   0   0   0   0   0   0   0   0       1   \n",
       "43346  10561  ...   0   0   0   0   0   0   0   0       1   \n",
       "43347  26083  ...   0   0   0   0   0   0   0   0       1   \n",
       "43348  39125  ...   0   0   0   0   0   0   0   0       1   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[43349 rows x 66 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copy_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>56</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 66 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8   \n",
       "0      101   10110     175  78653     189   25285   15976  40840    276  \\\n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  56  57  58  59  60  61  62  63  labels   \n",
       "0      31623  ...   0   0   0   0   0   0   0   0       0  \\\n",
       "1      23763  ...   0   0   0   0   0   0   0   0       0   \n",
       "2      76686  ...   0   0   0   0   0   0   0   0       0   \n",
       "3      10147  ...   0   0   0   0   0   0   0   0       0   \n",
       "4      13878  ...   0   0   0   0   0   0   0   0       0   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..  ..     ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0   0       1   \n",
       "43345    276  ...   0   0   0   0   0   0   0   0       1   \n",
       "43346  10561  ...   0   0   0   0   0   0   0   0       1   \n",
       "43347  26083  ...   0   0   0   0   0   0   0   0       1   \n",
       "43348  39125  ...   0   0   0   0   0   0   0   0       1   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[43349 rows x 66 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = embeding_df.tweet.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df=embeding_df.drop(['tweet'], axis=1)\n",
    "copy_df=copy_df.drop(['tweet'], axis=1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# isolation forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import IsolationForest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sebit/anaconda3/envs/dl_env/lib/python3.9/site-packages/sklearn/base.py:439: UserWarning: X does not have valid feature names, but IsolationForest was fitted with feature names\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "isf = IsolationForest(contamination=0.04)\n",
    "isf.fit(embeding_df)\n",
    "# Predictions\n",
    "predictions = isf.predict(embeding_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>iso_forest_scores</th>\n",
       "      <th>iso_forest_outliers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>43349.0</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.00000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "      <td>43349.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>101.0</td>\n",
       "      <td>27403.389559</td>\n",
       "      <td>29588.353803</td>\n",
       "      <td>26720.445131</td>\n",
       "      <td>27755.110106</td>\n",
       "      <td>27346.753628</td>\n",
       "      <td>27713.189255</td>\n",
       "      <td>27295.717687</td>\n",
       "      <td>27136.227410</td>\n",
       "      <td>26812.611156</td>\n",
       "      <td>...</td>\n",
       "      <td>4868.917184</td>\n",
       "      <td>4813.145309</td>\n",
       "      <td>4733.38919</td>\n",
       "      <td>4389.068375</td>\n",
       "      <td>4297.575723</td>\n",
       "      <td>4176.437080</td>\n",
       "      <td>17.000392</td>\n",
       "      <td>0.417957</td>\n",
       "      <td>0.135546</td>\n",
       "      <td>0.919998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>27382.274693</td>\n",
       "      <td>27727.688965</td>\n",
       "      <td>26455.267691</td>\n",
       "      <td>27026.611068</td>\n",
       "      <td>26799.753823</td>\n",
       "      <td>27021.950023</td>\n",
       "      <td>26761.847936</td>\n",
       "      <td>26820.810219</td>\n",
       "      <td>26720.480625</td>\n",
       "      <td>...</td>\n",
       "      <td>15312.358275</td>\n",
       "      <td>15491.136511</td>\n",
       "      <td>15387.09038</td>\n",
       "      <td>14617.253040</td>\n",
       "      <td>14643.580886</td>\n",
       "      <td>14405.397208</td>\n",
       "      <td>38.013945</td>\n",
       "      <td>0.493229</td>\n",
       "      <td>0.066701</td>\n",
       "      <td>0.391927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>101.0</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.140643</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>101.0</td>\n",
       "      <td>10357.000000</td>\n",
       "      <td>10506.000000</td>\n",
       "      <td>10323.000000</td>\n",
       "      <td>10361.000000</td>\n",
       "      <td>10350.000000</td>\n",
       "      <td>10369.000000</td>\n",
       "      <td>10347.000000</td>\n",
       "      <td>10330.000000</td>\n",
       "      <td>10323.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.089100</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>101.0</td>\n",
       "      <td>18856.000000</td>\n",
       "      <td>16263.000000</td>\n",
       "      <td>13587.000000</td>\n",
       "      <td>14918.000000</td>\n",
       "      <td>14753.000000</td>\n",
       "      <td>15090.000000</td>\n",
       "      <td>14777.000000</td>\n",
       "      <td>14753.000000</td>\n",
       "      <td>14110.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.161505</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>101.0</td>\n",
       "      <td>41079.000000</td>\n",
       "      <td>40762.000000</td>\n",
       "      <td>35943.000000</td>\n",
       "      <td>37820.000000</td>\n",
       "      <td>36544.000000</td>\n",
       "      <td>37820.000000</td>\n",
       "      <td>36723.000000</td>\n",
       "      <td>36544.000000</td>\n",
       "      <td>36445.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.189511</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>101.0</td>\n",
       "      <td>110744.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>111720.000000</td>\n",
       "      <td>111720.000000</td>\n",
       "      <td>111720.000000</td>\n",
       "      <td>111720.000000</td>\n",
       "      <td>111720.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.00000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>110966.000000</td>\n",
       "      <td>102.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.216831</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0              1              2              3              4   \n",
       "count  43349.0   43349.000000   43349.000000   43349.000000   43349.000000  \\\n",
       "mean     101.0   27403.389559   29588.353803   26720.445131   27755.110106   \n",
       "std        0.0   27382.274693   27727.688965   26455.267691   27026.611068   \n",
       "min      101.0     100.000000     100.000000       0.000000       0.000000   \n",
       "25%      101.0   10357.000000   10506.000000   10323.000000   10361.000000   \n",
       "50%      101.0   18856.000000   16263.000000   13587.000000   14918.000000   \n",
       "75%      101.0   41079.000000   40762.000000   35943.000000   37820.000000   \n",
       "max      101.0  110744.000000  110966.000000  110966.000000  110966.000000   \n",
       "\n",
       "                   5              6              7              8   \n",
       "count   43349.000000   43349.000000   43349.000000   43349.000000  \\\n",
       "mean    27346.753628   27713.189255   27295.717687   27136.227410   \n",
       "std     26799.753823   27021.950023   26761.847936   26820.810219   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%     10350.000000   10369.000000   10347.000000   10330.000000   \n",
       "50%     14753.000000   15090.000000   14777.000000   14753.000000   \n",
       "75%     36544.000000   37820.000000   36723.000000   36544.000000   \n",
       "max    111720.000000  111720.000000  111720.000000  111720.000000   \n",
       "\n",
       "                   9  ...             57             58            59   \n",
       "count   43349.000000  ...   43349.000000   43349.000000   43349.00000  \\\n",
       "mean    26812.611156  ...    4868.917184    4813.145309    4733.38919   \n",
       "std     26720.480625  ...   15312.358275   15491.136511   15387.09038   \n",
       "min         0.000000  ...       0.000000       0.000000       0.00000   \n",
       "25%     10323.000000  ...       0.000000       0.000000       0.00000   \n",
       "50%     14110.000000  ...       0.000000       0.000000       0.00000   \n",
       "75%     36445.000000  ...       0.000000       0.000000       0.00000   \n",
       "max    111720.000000  ...  110966.000000  110966.000000  110966.00000   \n",
       "\n",
       "                  60             61             62            63   \n",
       "count   43349.000000   43349.000000   43349.000000  43349.000000  \\\n",
       "mean     4389.068375    4297.575723    4176.437080     17.000392   \n",
       "std     14617.253040   14643.580886   14405.397208     38.013945   \n",
       "min         0.000000       0.000000       0.000000      0.000000   \n",
       "25%         0.000000       0.000000       0.000000      0.000000   \n",
       "50%         0.000000       0.000000       0.000000      0.000000   \n",
       "75%         0.000000       0.000000       0.000000      0.000000   \n",
       "max    110966.000000  110966.000000  110966.000000    102.000000   \n",
       "\n",
       "             labels  iso_forest_scores  iso_forest_outliers  \n",
       "count  43349.000000       43349.000000         43349.000000  \n",
       "mean       0.417957           0.135546             0.919998  \n",
       "std        0.493229           0.066701             0.391927  \n",
       "min        0.000000          -0.140643            -1.000000  \n",
       "25%        0.000000           0.089100             1.000000  \n",
       "50%        0.000000           0.161505             1.000000  \n",
       "75%        1.000000           0.189511             1.000000  \n",
       "max        1.000000           0.216831             1.000000  \n",
       "\n",
       "[8 rows x 67 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract scores\n",
    "embeding_df[\"iso_forest_scores\"] = isf.decision_function(embeding_df)\n",
    "# Extract predictions\n",
    "embeding_df[\"iso_forest_outliers\"] = predictions\n",
    "# Describe the dataframe\n",
    "embeding_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>57</th>\n",
       "      <th>58</th>\n",
       "      <th>59</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>iso_forest_scores</th>\n",
       "      <th>iso_forest_outliers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.189202</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.181234</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166332</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.151816</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.184008</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.079412</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.118245</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138229</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.181065</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085161</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43349 rows × 67 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8   \n",
       "0      101   10110     175  78653     189   25285   15976  40840    276  \\\n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  57  58  59  60  61  62  63  labels  iso_forest_scores   \n",
       "0      31623  ...   0   0   0   0   0   0   0       0           0.189202  \\\n",
       "1      23763  ...   0   0   0   0   0   0   0       0           0.181234   \n",
       "2      76686  ...   0   0   0   0   0   0   0       0           0.166332   \n",
       "3      10147  ...   0   0   0   0   0   0   0       0           0.151816   \n",
       "4      13878  ...   0   0   0   0   0   0   0       0           0.184008   \n",
       "...      ...  ...  ..  ..  ..  ..  ..  ..  ..     ...                ...   \n",
       "43344  92067  ...   0   0   0   0   0   0   0       1           0.079412   \n",
       "43345    276  ...   0   0   0   0   0   0   0       1           0.118245   \n",
       "43346  10561  ...   0   0   0   0   0   0   0       1           0.138229   \n",
       "43347  26083  ...   0   0   0   0   0   0   0       1           0.181065   \n",
       "43348  39125  ...   0   0   0   0   0   0   0       1           0.085161   \n",
       "\n",
       "       iso_forest_outliers  \n",
       "0                       No  \n",
       "1                       No  \n",
       "2                       No  \n",
       "3                       No  \n",
       "4                       No  \n",
       "...                    ...  \n",
       "43344                   No  \n",
       "43345                   No  \n",
       "43346                   No  \n",
       "43347                   No  \n",
       "43348                   No  \n",
       "\n",
       "[43349 rows x 67 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace \"-1\" with \"Yes\" and \"1\" with \"No\"\n",
    "embeding_df['iso_forest_outliers'] = embeding_df['iso_forest_outliers'].replace([-1, 1], [\"Yes\", \"No\"])\n",
    "# Print the first 5 firms\n",
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "iso_forest_outliers\n",
       "False    43349\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(embeding_df['iso_forest_outliers']=='YES').value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lof"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from numpy import quantile, where, random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model\n",
    "clf = LocalOutlierFactor(n_neighbors=20, contamination=0.1)\n",
    "out=clf.fit_predict(copy_df)\n",
    "# Predictions\n",
    "lof = clf.negative_outlier_factor_\n",
    "embeding_df[\"lof_outliers\"] = lof\n",
    "embeding_df[\"outliers\"]= out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df['outliers'] = embeding_df['outliers'].replace([-1, 1], [\"Yes\", \"No\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df['tweet']=data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=embeding_df[embeding_df['iso_forest_outliers']=='Yes' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df.drop(x.loc[x['outliers']=='Yes' ].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>60</th>\n",
       "      <th>61</th>\n",
       "      <th>62</th>\n",
       "      <th>63</th>\n",
       "      <th>labels</th>\n",
       "      <th>iso_forest_scores</th>\n",
       "      <th>iso_forest_outliers</th>\n",
       "      <th>lof_outliers</th>\n",
       "      <th>outliers</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>101</td>\n",
       "      <td>10110</td>\n",
       "      <td>175</td>\n",
       "      <td>78653</td>\n",
       "      <td>189</td>\n",
       "      <td>25285</td>\n",
       "      <td>15976</td>\n",
       "      <td>40840</td>\n",
       "      <td>276</td>\n",
       "      <td>31623</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.189202</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.209681</td>\n",
       "      <td>No</td>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>101</td>\n",
       "      <td>11589</td>\n",
       "      <td>10706</td>\n",
       "      <td>10713</td>\n",
       "      <td>10794</td>\n",
       "      <td>94698</td>\n",
       "      <td>30668</td>\n",
       "      <td>24883</td>\n",
       "      <td>117</td>\n",
       "      <td>23763</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.181234</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.107479</td>\n",
       "      <td>No</td>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101</td>\n",
       "      <td>148</td>\n",
       "      <td>30471</td>\n",
       "      <td>10774</td>\n",
       "      <td>13785</td>\n",
       "      <td>13779</td>\n",
       "      <td>33642</td>\n",
       "      <td>14399</td>\n",
       "      <td>48271</td>\n",
       "      <td>76686</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166332</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.202529</td>\n",
       "      <td>No</td>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101</td>\n",
       "      <td>19319</td>\n",
       "      <td>16724</td>\n",
       "      <td>10118</td>\n",
       "      <td>10107</td>\n",
       "      <td>78323</td>\n",
       "      <td>12407</td>\n",
       "      <td>38959</td>\n",
       "      <td>22934</td>\n",
       "      <td>10147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.151816</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.216599</td>\n",
       "      <td>No</td>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101</td>\n",
       "      <td>30932</td>\n",
       "      <td>58706</td>\n",
       "      <td>58054</td>\n",
       "      <td>44907</td>\n",
       "      <td>10224</td>\n",
       "      <td>106583</td>\n",
       "      <td>10288</td>\n",
       "      <td>12524</td>\n",
       "      <td>13878</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.184008</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.188488</td>\n",
       "      <td>No</td>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>101</td>\n",
       "      <td>20065</td>\n",
       "      <td>10161</td>\n",
       "      <td>115</td>\n",
       "      <td>115</td>\n",
       "      <td>103784</td>\n",
       "      <td>10774</td>\n",
       "      <td>21388</td>\n",
       "      <td>10245</td>\n",
       "      <td>92067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.079412</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.196769</td>\n",
       "      <td>No</td>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>101</td>\n",
       "      <td>139</td>\n",
       "      <td>80839</td>\n",
       "      <td>24109</td>\n",
       "      <td>13406</td>\n",
       "      <td>18985</td>\n",
       "      <td>16285</td>\n",
       "      <td>10163</td>\n",
       "      <td>11062</td>\n",
       "      <td>276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.118245</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.108304</td>\n",
       "      <td>No</td>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>101</td>\n",
       "      <td>105549</td>\n",
       "      <td>102635</td>\n",
       "      <td>10140</td>\n",
       "      <td>26943</td>\n",
       "      <td>11499</td>\n",
       "      <td>110516</td>\n",
       "      <td>21899</td>\n",
       "      <td>11861</td>\n",
       "      <td>10561</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138229</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.307328</td>\n",
       "      <td>No</td>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>101</td>\n",
       "      <td>81424</td>\n",
       "      <td>26398</td>\n",
       "      <td>92017</td>\n",
       "      <td>109620</td>\n",
       "      <td>10941</td>\n",
       "      <td>76010</td>\n",
       "      <td>10115</td>\n",
       "      <td>19830</td>\n",
       "      <td>26083</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.181065</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.127932</td>\n",
       "      <td>No</td>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>101</td>\n",
       "      <td>39774</td>\n",
       "      <td>11127</td>\n",
       "      <td>45989</td>\n",
       "      <td>24596</td>\n",
       "      <td>11933</td>\n",
       "      <td>170</td>\n",
       "      <td>17145</td>\n",
       "      <td>10710</td>\n",
       "      <td>39125</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.085161</td>\n",
       "      <td>No</td>\n",
       "      <td>-1.286323</td>\n",
       "      <td>No</td>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43029 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         0       1       2      3       4       5       6      7      8   \n",
       "0      101   10110     175  78653     189   25285   15976  40840    276  \\\n",
       "1      101   11589   10706  10713   10794   94698   30668  24883    117   \n",
       "2      101     148   30471  10774   13785   13779   33642  14399  48271   \n",
       "3      101   19319   16724  10118   10107   78323   12407  38959  22934   \n",
       "4      101   30932   58706  58054   44907   10224  106583  10288  12524   \n",
       "...    ...     ...     ...    ...     ...     ...     ...    ...    ...   \n",
       "43344  101   20065   10161    115     115  103784   10774  21388  10245   \n",
       "43345  101     139   80839  24109   13406   18985   16285  10163  11062   \n",
       "43346  101  105549  102635  10140   26943   11499  110516  21899  11861   \n",
       "43347  101   81424   26398  92017  109620   10941   76010  10115  19830   \n",
       "43348  101   39774   11127  45989   24596   11933     170  17145  10710   \n",
       "\n",
       "           9  ...  60  61  62  63  labels  iso_forest_scores   \n",
       "0      31623  ...   0   0   0   0       0           0.189202  \\\n",
       "1      23763  ...   0   0   0   0       0           0.181234   \n",
       "2      76686  ...   0   0   0   0       0           0.166332   \n",
       "3      10147  ...   0   0   0   0       0           0.151816   \n",
       "4      13878  ...   0   0   0   0       0           0.184008   \n",
       "...      ...  ...  ..  ..  ..  ..     ...                ...   \n",
       "43344  92067  ...   0   0   0   0       1           0.079412   \n",
       "43345    276  ...   0   0   0   0       1           0.118245   \n",
       "43346  10561  ...   0   0   0   0       1           0.138229   \n",
       "43347  26083  ...   0   0   0   0       1           0.181065   \n",
       "43348  39125  ...   0   0   0   0       1           0.085161   \n",
       "\n",
       "       iso_forest_outliers  lof_outliers  outliers   \n",
       "0                       No     -1.209681        No  \\\n",
       "1                       No     -1.107479        No   \n",
       "2                       No     -1.202529        No   \n",
       "3                       No     -1.216599        No   \n",
       "4                       No     -1.188488        No   \n",
       "...                    ...           ...       ...   \n",
       "43344                   No     -1.196769        No   \n",
       "43345                   No     -1.108304        No   \n",
       "43346                   No     -1.307328        No   \n",
       "43347                   No     -1.127932        No   \n",
       "43348                   No     -1.286323        No   \n",
       "\n",
       "                                                   tweet  \n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...  \n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...  \n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...  \n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...  \n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı  \n",
       "...                                                  ...  \n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...  \n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...  \n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...  \n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...  \n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...  \n",
       "\n",
       "[43029 rows x 70 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeding_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embeding_df.drop(embeding_df.loc[embeding_df['outliers']=='Yes' ].index, inplace=True)\n",
    "# embeding_df.drop(embeding_df.loc[embeding_df['iso_forest_outliers']=='Yes' ].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iso_df=embeding_df[embeding_df['iso_forest_outliers']=='Yes' ]\n",
    "# embeding_df.drop(embeding_df.loc[embeding_df['iso_forest_outliers']=='Yes' ].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lof_df=embeding_df[embeding_df['outliers']=='Yes' ]\n",
    "# embeding_df.drop(embeding_df.loc[embeding_df['outliers']=='Yes' ].index, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iso_df['labels']=iso_df['labels'].replace({0: 1, 1: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iso_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lof_df['labels']=lof_df['labels'].replace({0: 1, 1: 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lof_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# x=pd.concat([lof_df,iso_df], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embeding_df=pd.concat([x,embeding_df], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embeding_df.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embeding_df=embeding_df.drop(['iso_forest_scores', 'iso_forest_outliers','lof_outliers','outliers'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "embeding_df['0'] = embeding_df[embeding_df.columns[:-1]].apply(lambda x: ','.join(x.dropna().astype(str)),axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame()\n",
    "df['tweet']=embeding_df['tweet']\n",
    "df['subtas_a']=embeding_df['labels']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet</th>\n",
       "      <th>subtas_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en güzel uyuyan insan ödülü jeon jungkook'a g...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Mekanı cennet olsun, saygılar sayın avukatımı...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kızlar aranızda kas yığını beylere düşenler ol...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Biraz ders çalışayım. Tembellik ve uyku düşman...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Trezeguet yerine El Sharawy daha iyi olmaz mı</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43344</th>\n",
       "      <td>Hil**adamlar kesinlikle kelimeleri anlamıyorla...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43345</th>\n",
       "      <td>Böyle piçlerin çok erken ölmemelerini ve çok f...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43346</th>\n",
       "      <td>Turgay denilen bu holigonda bir sorun yok, gur...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43347</th>\n",
       "      <td>Umarım ülkenin düşük zekadan kurtulması ilgile...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43348</th>\n",
       "      <td>CHP sandıkları bırakmaz, üzerine oturur, bir c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>43029 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   tweet  subtas_a\n",
       "0       en güzel uyuyan insan ödülü jeon jungkook'a g...         0\n",
       "1       Mekanı cennet olsun, saygılar sayın avukatımı...         0\n",
       "2      Kızlar aranızda kas yığını beylere düşenler ol...         0\n",
       "3      Biraz ders çalışayım. Tembellik ve uyku düşman...         0\n",
       "4          Trezeguet yerine El Sharawy daha iyi olmaz mı         0\n",
       "...                                                  ...       ...\n",
       "43344  Hil**adamlar kesinlikle kelimeleri anlamıyorla...         1\n",
       "43345  Böyle piçlerin çok erken ölmemelerini ve çok f...         1\n",
       "43346  Turgay denilen bu holigonda bir sorun yok, gur...         1\n",
       "43347  Umarım ülkenin düşük zekadan kurtulması ilgile...         1\n",
       "43348  CHP sandıkları bırakmaz, üzerine oturur, bir c...         1\n",
       "\n",
       "[43029 rows x 2 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('inverse_outliers.csv')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('int_2_outliers.csv')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dl_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}