push the changes

.gitignore

@@ -1,2 +1,5 @@
 *.pyc
-*.png
+*.png
+AQI*/
+*.parquet
+*.zip

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "cSpell.words": ["dataframe"]
+}

Google_Talk_chatbot.ipynb

@@ -0,0 +1,697 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import streamlit as st\n",
+    "import os\n",
+    "import pandas as pd\n",
+    "import random\n",
+    "from os.path import join\n",
+    "from datetime import datetime\n",
+    "# from src import decorate_with_code, show_response, get_from_user\n",
+    "from dotenv import load_dotenv\n",
+    "from langchain_groq.chat_models import ChatGroq\n",
+    "from langchain_mistralai import ChatMistralAI\n",
+    "from langchain_google_genai import GoogleGenerativeAI,GoogleGenerativeAIEmbeddings\n",
+    "from huggingface_hub import HfApi\n",
+    "load_dotenv()\n",
+    "\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>prompt</th>\n",
+       "      <th>Desired Answer</th>\n",
+       "      <th>Category</th>\n",
+       "      <th>llama3_answer_1721726247</th>\n",
+       "      <th>llama3_score_1721726247</th>\n",
+       "      <th>mixtral_answer_1721726407</th>\n",
+       "      <th>mixtral_score_1721726407</th>\n",
+       "      <th>gemma_answer_1721726499</th>\n",
+       "      <th>gemma_score_1721726499</th>\n",
+       "      <th>Codestral Mamba_answer_1721759526</th>\n",
+       "      <th>Codestral Mamba_score_1721759526</th>\n",
+       "      <th>Codestral_answer_1721759762</th>\n",
+       "      <th>Codestral_score_1721759762</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Which month has the highest average PM2.5 in 2...</td>\n",
+       "      <td>January</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>The month with the highest average PM2.5 in 20...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>The month with the highest average PM2.5 in 20...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>The highest average PM2.5 in 2023 for Mumbai w...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>content='To find the month with the highest av...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>The month with the highest average PM2.5 in 20...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Which month generally has the highest pollution?</td>\n",
+       "      <td>November</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>The month with the highest pollution is 11 wit...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>The month with the highest pollution (on avera...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>The month with the highest average PM2.5 is No...</td>\n",
+       "      <td>True</td>\n",
+       "      <td>content='To find out which month generally has...</td>\n",
+       "      <td>False</td>\n",
+       "      <td>The month with the highest pollution is Novemb...</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                              prompt Desired Answer  \\\n",
+       "0  Which month has the highest average PM2.5 in 2...        January   \n",
+       "1   Which month generally has the highest pollution?       November   \n",
+       "\n",
+       "   Category                            llama3_answer_1721726247  \\\n",
+       "0        NaN  The month with the highest average PM2.5 in 20...   \n",
+       "1        NaN  The month with the highest pollution is 11 wit...   \n",
+       "\n",
+       "  llama3_score_1721726247                          mixtral_answer_1721726407  \\\n",
+       "0                    True  The month with the highest average PM2.5 in 20...   \n",
+       "1                    True  The month with the highest pollution (on avera...   \n",
+       "\n",
+       "  mixtral_score_1721726407                            gemma_answer_1721726499  \\\n",
+       "0                     True  The highest average PM2.5 in 2023 for Mumbai w...   \n",
+       "1                     True  The month with the highest average PM2.5 is No...   \n",
+       "\n",
+       "  gemma_score_1721726499                  Codestral Mamba_answer_1721759526  \\\n",
+       "0                   True  content='To find the month with the highest av...   \n",
+       "1                   True  content='To find out which month generally has...   \n",
+       "\n",
+       "  Codestral Mamba_score_1721759526  \\\n",
+       "0                             True   \n",
+       "1                            False   \n",
+       "\n",
+       "                         Codestral_answer_1721759762  \\\n",
+       "0  The month with the highest average PM2.5 in 20...   \n",
+       "1  The month with the highest pollution is Novemb...   \n",
+       "\n",
+       "  Codestral_score_1721759762  \n",
+       "0                       True  \n",
+       "1                       True  "
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "prompts = pd.read_csv(\"prompts.csv\")\n",
+    "prompts.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Date                                      datetime64[ns]\n",
+      "City                                              object\n",
+      "AQI                                              float64\n",
+      "Pollutant                                         object\n",
+      "Air Quality                                       object\n",
+      "Based on number of monitoring stations           float64\n",
+      "State                                             object\n",
+      "Date_City                                         object\n",
+      "dtype: object\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Date</th>\n",
+       "      <th>City</th>\n",
+       "      <th>AQI</th>\n",
+       "      <th>Pollutant</th>\n",
+       "      <th>Air Quality</th>\n",
+       "      <th>Based on number of monitoring stations</th>\n",
+       "      <th>State</th>\n",
+       "      <th>Date_City</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2016-01-01</td>\n",
+       "      <td>Agartala</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>2016-01-01</td>\n",
+       "      <td>Agra</td>\n",
+       "      <td>417.0</td>\n",
+       "      <td>PM\\n2.5</td>\n",
+       "      <td>Severe</td>\n",
+       "      <td>1.0</td>\n",
+       "      <td>Uttar Pradesh</td>\n",
+       "      <td>2016-01-01_Agra</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        Date      City    AQI Pollutant Air Quality  \\\n",
+       "0 2016-01-01  Agartala    NaN      None        None   \n",
+       "1 2016-01-01      Agra  417.0   PM\\n2.5      Severe   \n",
+       "\n",
+       "   Based on number of monitoring stations          State        Date_City  \n",
+       "0                                     NaN           None             None  \n",
+       "1                                     1.0  Uttar Pradesh  2016-01-01_Agra  "
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = pd.read_parquet(\"AQI_data.parquet\")\n",
+    "print(data.dtypes)\n",
+    "data.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_full_prompt(question):\n",
+    "    return f\"\"\"You are a data scientist who is good with tools such as pandas, numpy and matplotlib. You are also an expert in air quality. You\n",
+    "can access daily AQI `data` and you have to complete a code provided by me based on my question. `data` is a pandas DataFrame and has the following columns and data types:\n",
+    "\n",
+    "Date: date, Date of the `AQI` data\n",
+    "City: string, Name of the city where the `AQI` was recorded\n",
+    "State: string, Name of the state where `City` is located\n",
+    "AQI: float, AQI value\n",
+    "Air Quality: string, Air quality category from [\"Satisfactory\", \"Moderate\", \"Good\", \"Poor\", \"Very Poor\", \"Severe\"] based on `AQI` value\n",
+    "\n",
+    "Now, my question is: \"{question}\"\n",
+    "\n",
+    "Complete the code below to answer my question:\n",
+    "\n",
+    "```python\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "data = pd.read_parquet(\"AQI_data.parquet\")\"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 35,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "You are a data scientist who is good with tools such as pandas, numpy and matplotlib. You are also an expert in air quality. You\n",
+      "can access daily AQI `data` and you have to complete a code provided by me based on my question. `data` is a pandas DataFrame and has the following columns and data types:\n",
+      "\n",
+      "Date: date, Date of the `AQI` data\n",
+      "City: string, Name of the city where the `AQI` was recorded\n",
+      "State: string, Name of the state where `City` is located\n",
+      "AQI: float, AQI value\n",
+      "Air Quality: string, Air quality category from [\"Satisfactory\", \"Moderate\", \"Good\", \"Poor\", \"Very Poor\", \"Severe\"] based on `AQI` value\n",
+      "\n",
+      "Now, my question is: \"Which month has the highest average PM2.5 in 2023 for Mumbai?\"\n",
+      "\n",
+      "Complete the code below to answer my question:\n",
+      "\n",
+      "```python\n",
+      "import pandas as pd\n",
+      "import numpy as np\n",
+      "\n",
+      "data = pd.read_parquet(\"AQI_data.parquet\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(get_full_prompt(prompts.prompt[0]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_gemini_response(prompt, model):\n",
+    "    return GoogleGenerativeAI(model=model, google_api_key=os.environ.get(\"GOOGLE_API_KEY\"), temperature=0).invoke(prompt)\n",
+    "\n",
+    "def get_groq_response(prompt, model):\n",
+    "    return ChatGroq(model=model, api_key=os.environ.get(\"GROQ_API_KEY\"), temperature=0).invoke(prompt).content\n",
+    "\n",
+    "llms = {\"gemini-pro\": lambda prompt: get_gemini_response(prompt, \"gemini-pro\"), \"groq_gemma-7b-it\": lambda prompt: get_groq_response(prompt, \"gemma-7b-it\"), \"groq_llama-3.2-90b-text-preview\": lambda prompt: get_groq_response(prompt, \"llama-3.2-90b-text-preview\")}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('Which month generally has the highest pollution?', 'November')"
+      ]
+     },
+     "execution_count": 75,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "i = 1\n",
+    "prompts.prompt[i], prompts['Desired Answer'][i]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "llm = \"groq_llama-3.2-90b-text-preview\"\n",
+    "full_prompt = get_full_prompt(\"Which city has the highest AQI value consistently over the years?\")\n",
+    "answer = llms[llm](full_prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "You are a data scientist who is good with tools such as pandas, numpy and matplotlib. You are also an expert in air quality. You\n",
+      "can access daily AQI `data` and you have to complete a code provided by me based on my question. `data` is a pandas DataFrame and has the following columns and data types:\n",
+      "\n",
+      "Date: date, Date of the `AQI` data\n",
+      "City: string, Name of the city where the `AQI` was recorded\n",
+      "State: string, Name of the state where `City` is located\n",
+      "AQI: float, AQI value\n",
+      "Air Quality: string, Air quality category from [\"Satisfactory\", \"Moderate\", \"Good\", \"Poor\", \"Very Poor\", \"Severe\"] based on `AQI` value\n",
+      "\n",
+      "Now, my question is: \"Which city has the highest AQI value consistently over the years?\"\n",
+      "\n",
+      "Complete the code below to answer my question:\n",
+      "\n",
+      "```python\n",
+      "import pandas as pd\n",
+      "import numpy as np\n",
+      "\n",
+      "data = pd.read_parquet(\"AQI_data.parquet\")\n",
+      "####################################################################################################\n",
+      "import pandas as pd\n",
+      "import numpy as np\n",
+      "import matplotlib.pyplot as plt\n",
+      "\n",
+      "# Load the data\n",
+      "data = pd.read_parquet(\"AQI_data.parquet\")\n",
+      "\n",
+      "# Convert 'Date' column to datetime and extract the year\n",
+      "data['Date'] = pd.to_datetime(data['Date'])\n",
+      "data['Year'] = data['Date'].dt.year\n",
+      "\n",
+      "# Group by 'City' and 'Year', and calculate the average AQI value\n",
+      "avg_aqi = data.groupby(['City', 'Year'])['AQI'].mean().reset_index()\n",
+      "\n",
+      "# Group by 'City' and calculate the average AQI value over the years\n",
+      "avg_aqi_over_years = avg_aqi.groupby('City')['AQI'].mean().reset_index()\n",
+      "\n",
+      "# Find the city with the highest average AQI value\n",
+      "city_with_highest_aqi = avg_aqi_over_years.loc[avg_aqi_over_years['AQI'].idxmax()]\n",
+      "\n",
+      "print(f\"The city with the highest AQI value consistently over the years is {city_with_highest_aqi['City']} with an average AQI value of {city_with_highest_aqi['AQI']:.2f}\")\n",
+      "\n",
+      "# Plot the top 10 cities with the highest average AQI values\n",
+      "top_10_cities = avg_aqi_over_years.nlargest(10, 'AQI')\n",
+      "plt.figure(figsize=(10, 6))\n",
+      "plt.bar(top_10_cities['City'], top_10_cities['AQI'])\n",
+      "plt.xlabel('City')\n",
+      "plt.ylabel('Average AQI Value')\n",
+      "plt.title('Top 10 Cities with the Highest Average AQI Values')\n",
+      "plt.xticks(rotation=90)\n",
+      "plt.tight_layout()\n",
+      "plt.show()\n",
+      "\n",
+      "####################################################################################################\n",
+      "The city with the highest AQI value consistently over the years is Jharsuguda with an average AQI value of 282.00\n"
+     ]
+    }
+   ],
+   "source": [
+    "import re\n",
+    "code = re.search(r\"```python\\n(.*)```\", answer, re.DOTALL).group(1)\n",
+    "print(full_prompt)\n",
+    "print(\"#\" * 100)\n",
+    "print(code)\n",
+    "print(\"#\" * 100)\n",
+    "exec(code)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import streamlit as st\n",
+    "# import os\n",
+    "# import pandas as pd\n",
+    "# import random\n",
+    "# from os.path import join\n",
+    "# from datetime import datetime\n",
+    "# from src import decorate_with_code, show_response, get_from_user\n",
+    "# from dotenv import load_dotenv\n",
+    "# from langchain_groq.chat_models import ChatGroq\n",
+    "# from langchain_mistralai import ChatMistralAI\n",
+    "# from huggingface_hub import HfApi\n",
+    "# st.set_page_config(layout=\"wide\")\n",
+    "\n",
+    "# ### Extract data.zip\n",
+    "# if not os.path.exists(\"data/1\"):\n",
+    "#     os.system(\"unzip data.zip\")\n",
+    "\n",
+    "# # Load environment variables : Groq and Hugging Face API keys\n",
+    "# load_dotenv()\n",
+    "# Groq_Token = os.environ[\"GROQ_API_KEY\"]\n",
+    "# CODESTRAL_API_KEY = os.environ[\"CODESTRAL_API_KEY\"]\n",
+    "# hf_token = os.environ[\"HF_TOKEN\"]\n",
+    "# models = {\"llama3\":\"llama3-70b-8192\",\"mixtral\": \"mixtral-8x7b-32768\", \"llama2\": \"llama2-70b-4096\", \"gemma\": \"gemma-7b-it\"}\n",
+    "# groq_models = {\"llama3-70b\": \"llama3-70b-8192\", \"mixtral\": \"mixtral-8x7b-32768\", \"gemma-7b\": \"gemma-7b-it\",\"llama3.1-70b\":\"llama-3.1-70b-versatile\",\"llama3-8b\":\"llama3-8b-8192\",\"llama3.1-8b\":\"llama-3.1-8b-instant\",\"gemma-9b\":\"gemma2-9b-it\"}\n",
+    "# mistral_models = {\"Codestral Mamba\" : \"open-codestral-mamba\", \"Codestral\" : \"codestral-latest\",\"Mistral 7B\":\"open-mistral-7b\"}\n",
+    "# groq_model_list = list(groq_models.keys())\n",
+    "# mistral_model_list = list(mistral_models.keys())\n",
+    "\n",
+    "# self_path = os.path.dirname(os.path.abspath(__file__))\n",
+    "\n",
+    "\n",
+    "# def generate_template(prompt):\n",
+    "#     df_check = pd.read_csv(\"Data.csv\")\n",
+    "#     df_check[\"Timestamp\"] = pd.to_datetime(df_check[\"Timestamp\"])\n",
+    "#     df_check = df_check.head(5)\n",
+    "\n",
+    "#     new_line = \"\\n\"\n",
+    "\n",
+    "#     template = f\"\"\"```python\n",
+    "# import pandas as pd\n",
+    "# import matplotlib.pyplot as plt\n",
+    "\n",
+    "# df = pd.read_csv(\"Data.csv\")\n",
+    "# df[\"Timestamp\"] = pd.to_datetime(df[\"Timestamp\"])\n",
+    "\n",
+    "# # df.dtypes\n",
+    "# {new_line.join(map(lambda x: '# '+x, str(df_check.dtypes).split(new_line)))}\n",
+    "\n",
+    "# # {prompt.strip()}\n",
+    "# # <your code here>\n",
+    "\n",
+    "# #answer = \n",
+    "# ```\n",
+    "# \"\"\"\n",
+    "#     return template\n",
+    "\n",
+    "\n",
+    "# def generate_query(template):\n",
+    "    \n",
+    "#     query = f\"\"\"I have a pandas dataframe data of PM2.5.\n",
+    "# * The columns are 'Timestamp', 'station', 'PM2.5', 'address', 'city', 'latitude', 'longitude', and 'state'.\n",
+    "# * Frequency of Data is Daily.\n",
+    "# * `Pollution` generally means `PM2.5`.\n",
+    "# * PM2.5 guidelines: India: 60, WHO: 15.\n",
+    "# * Store the final answer in a global variable `answer`.\n",
+    "# * Always report the unit of the data. Example: `The average PM2.5 is 45.67 µg/m³`\n",
+    "\n",
+    "# Complete the following code.\n",
+    "\n",
+    "# {template}\n",
+    "# \"\"\"\n",
+    "#     return query\n",
+    "\n",
+    "\n",
+    "# def process_query(query, llm):\n",
+    "#     global answer\n",
+    "#     template = generate_template(query)\n",
+    "#     query = generate_query(template)\n",
+    "#     global code\n",
+    "#     global error\n",
+    "#     try:\n",
+    "#         answer = llm.invoke(query)\n",
+    "#         error = ''\n",
+    "#         code = f\"\"\"\n",
+    "#         {template.split(\"```python\")[1].split(\"```\")[0]}\n",
+    "#         {answer.content.split(\"```python\")[1].split(\"```\")[0]}\n",
+    "#         \"\"\"\n",
+    "#         # update variable `answer` when code is executed\n",
+    "#         exec(code,globals())\n",
+    "#     except Exception as e:\n",
+    "#         error = e\n",
+    "#         code = ''\n",
+    "#         answer = f\"Error: {e}\"\n",
+    "#     print(answer)\n",
+    "\n",
+    "\n",
+    "# # Using HTML and CSS to center the title\n",
+    "# st.write(\n",
+    "#     \"\"\"\n",
+    "#     <style>\n",
+    "#     .title {\n",
+    "#         text-align: center;\n",
+    "#         color: #17becf;\n",
+    "#     }\n",
+    "#     </style>\n",
+    "# \"\"\",\n",
+    "#     unsafe_allow_html=True,\n",
+    "# )\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "#     # Display images and text in three columns with specified ratios\n",
+    "# col1, col2, col3 = st.sidebar.columns((1.0, 2, 1.0))  \n",
+    "# with col2:\n",
+    "#     st.markdown(\"<h1 class='title'>Airchat</h1>\", unsafe_allow_html=True)\n",
+    "    \n",
+    "    \n",
+    "# model_name = st.sidebar.selectbox(\"Select LLM:\", groq_model_list + mistral_model_list)\n",
+    "\n",
+    "# questions = ['Custom Prompt']\n",
+    "# with open(join(self_path, \"questions.txt\")) as f:\n",
+    "#     questions += f.read().split(\"\\n\")\n",
+    "\n",
+    "# waiting_lines = (\"Thinking...\", \"Just a moment...\", \"Let me think...\", \"Working on it...\", \"Processing...\", \"Hold on...\", \"One moment...\", \"On it...\")\n",
+    "\n",
+    "\n",
+    "\n",
+    "# # Initialize chat history\n",
+    "# if \"responses\" not in st.session_state:\n",
+    "#     st.session_state.responses = []\n",
+    "  \n",
+    "\n",
+    "# # Display chat responses from history on app rerun\n",
+    "# print(\"#\"*10)\n",
+    "# for response_id, response in enumerate(st.session_state.responses):\n",
+    "#     status = show_response(st, response)\n",
+    "#     if response[\"role\"] == \"assistant\":\n",
+    "#         # feedback_key = f\"feedback_{int(response_id/2)}\"\n",
+    "#         print(\"response_id\", response_id)\n",
+    "        \n",
+    "#         error = response[\"error\"]\n",
+    "#         output = response[\"content\"]\n",
+    "#         last_prompt = response[\"last_prompt\"]\n",
+    "#         code = response[\"gen_code\"]\n",
+    "#         evaluation = response[\"evaluation\"]\n",
+    "        \n",
+    "                \n",
+    "        \n",
+    "# print(\"#\"*10)\n",
+    "\n",
+    "# show = True\n",
+    "# prompt = st.sidebar.selectbox(\"Select a Prompt:\", questions, key=\"prompt_key\")\n",
+    "# if prompt == 'Custom Prompt':\n",
+    "#     show = False\n",
+    "#     # React to user input\n",
+    "#     prompt = st.chat_input(\"Ask me anything about air quality!\", key=1000)\n",
+    "#     if prompt :\n",
+    "#         show = True\n",
+    "# else:\n",
+    "#     # placeholder for chat input\n",
+    "#     st.chat_input(\"Select 'Select a Prompt' -> 'Custom Prompt' in the sidebar to ask your own questions.\", key=1000, disabled=True)\n",
+    "\n",
+    "# if \"last_prompt\" in st.session_state:\n",
+    "#     last_prompt = st.session_state[\"last_prompt\"]\n",
+    "#     last_model_name = st.session_state[\"last_model_name\"]\n",
+    "#     if (prompt == last_prompt) and (model_name == last_model_name):\n",
+    "#         show = False\n",
+    "\n",
+    "# if prompt:\n",
+    "#     st.sidebar.info(\"Select 'Custom Prompt' to ask your own questions.\")\n",
+    "\n",
+    "#     if show:\n",
+    "#         # Add user input to chat history\n",
+    "#         user_response = get_from_user(prompt)\n",
+    "#         st.session_state.responses.append(user_response)\n",
+    "\n",
+    "#         # select random waiting line\n",
+    "#         with st.spinner(random.choice(waiting_lines)):\n",
+    "#             ran = False\n",
+    "#             for i in range(1):\n",
+    "#                 print(f\"Attempt {i+1}\")\n",
+    "#                 if model_name in groq_models:\n",
+    "#                     model_folder = \"Groq_\" + groq_models[model_name]\n",
+    "#                     llm = ChatGroq(model=groq_models[model_name], api_key=Groq_Token, temperature=0)\n",
+    "#                 else:\n",
+    "#                     model_folder = \"MistralAI_\" + mistral_models[model_name]\n",
+    "#                     llm = ChatMistralAI(model=mistral_models[model_name], api_key=CODESTRAL_API_KEY, temperature=0)\n",
+    "#                 print(llm)\n",
+    "#                 # llm = ChatGroq(model=models[model_name], api_key=os.getenv(\"GROQ_API\"), temperature=0)\n",
+    "\n",
+    "#                 df_check = pd.read_csv(\"Data.csv\")\n",
+    "#                 df_check[\"Timestamp\"] = pd.to_datetime(df_check[\"Timestamp\"])\n",
+    "#                 df_check = df_check.head(5)\n",
+    "\n",
+    "#                 new_line = \"\\n\"\n",
+    "\n",
+    "#                 parameters = {\"font.size\": 12,\"figure.dpi\": 600}\n",
+    "\n",
+    "#                 process_query(prompt, llm)\n",
+    "                \n",
+    "                \n",
+    "#                 # Read the questions from Questions.txt and find the index of the question if there is a match\n",
+    "#                 with open(join(\"questions.txt\")) as f:\n",
+    "#                     questions = f.read().split(\"\\n\")\n",
+    "#                     try:\n",
+    "#                         index = questions.index(prompt)\n",
+    "#                         index = index + 1\n",
+    "#                     except:\n",
+    "#                         index = None \n",
+    "#                 print(\"Index\",index)\n",
+    "#                 if type(index) == int:\n",
+    "#                     # Open folder data/index/llm_name and compare with evaluation.txt\n",
+    "#                     with open(join(\"data\", str(index), model_folder, \"evaluation.txt\")) as f:\n",
+    "#                         evaluation = f.read().strip()\n",
+    "#                     with open(join(\"data\", str(index), \"ground_truth\", \"answer.txt\")) as f:\n",
+    "#                         ground_truth = f.read().strip()\n",
+    "#                 else:\n",
+    "#                     evaluation = \"DK\"\n",
+    "#                     ground_truth = None \n",
+    "#                 response = {\"role\": \"assistant\", \"content\": answer, \"gen_code\": code, \"ex_code\": code, \"last_prompt\": prompt, \"error\": error,\"evaluation\": evaluation,\"ground_truth\": ground_truth}\n",
+    "\n",
+    "#                 if ran:\n",
+    "#                     break\n",
+    "            \n",
+    "#         # Append agent response to chat history\n",
+    "#         st.session_state.responses.append(response)\n",
+    "        \n",
+    "#         st.session_state['last_prompt'] = prompt\n",
+    "#         st.session_state['last_model_name'] = model_name\n",
+    "#         st.rerun()\n",
+    "        \n",
+    "        \n",
+    "\n",
+    "# # Display contact details with message\n",
+    "# st.sidebar.markdown(\"<hr>\", unsafe_allow_html=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "zeel_py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

app.py

@@ -5,13 +5,22 @@ import pandas as pd
 import random
 from os.path import join
 from datetime import datetime
-from src import preprocess_and_load_df, load_agent, ask_agent, decorate_with_code, show_response, get_from_user, load_smart_df, ask_question
+from src import (
+    preprocess_and_load_df,
+    load_agent,
+    ask_agent,
+    decorate_with_code,
+    show_response,
+    get_from_user,
+    load_smart_df,
+    ask_question,
+)
 from dotenv import load_dotenv
 from langchain_groq.chat_models import ChatGroq
 from langchain_google_genai import GoogleGenerativeAI
 from streamlit_feedback import streamlit_feedback
 from huggingface_hub import HfApi
-from datasets import load_dataset, get_dataset_config_info
+from datasets import load_dataset, get_dataset_config_info, Dataset
 from PIL import Image
 
 st.set_page_config(layout="wide")
@@ -21,7 +30,13 @@ load_dotenv()
 Groq_Token = os.environ["GROQ_API_KEY"]
 hf_token = os.environ["HF_TOKEN"]
 gemini_token = os.environ["GEMINI_TOKEN"]
-models = {"llama3":"llama3-70b-8192","mixtral": "mixtral-8x7b-32768", "llama2": "llama2-70b-4096", "gemma": "gemma-7b-it", "gemini-pro": "gemini-pro"}
+models = {
+    "llama3": "llama3-70b-8192",
+    "mixtral": "mixtral-8x7b-32768",
+    "llama2": "llama2-70b-4096",
+    "gemma": "gemma-7b-it",
+    "gemini-pro": "gemini-pro",
+}
 
 self_path = os.path.dirname(os.path.abspath(__file__))
 
@@ -40,10 +55,16 @@ st.write(
 )
 
 # Displaying the centered title
-st.markdown("<div style='text-align:center; padding: 20px;'>VayuBuddy makes pollution monitoring easier by bridging the gap between users and datasets.<br>No coding required—just meaningful insights at your fingertips!</div>", unsafe_allow_html=True)
+st.markdown(
+    "<div style='text-align:center; padding: 20px;'>VayuBuddy makes pollution monitoring easier by bridging the gap between users and datasets.<br>No coding required—just meaningful insights at your fingertips!</div>",
+    unsafe_allow_html=True,
+)
 
 # Center-aligned instruction text with bold formatting
-st.markdown("<div style='text-align:center;'>Choose a query from <b>Select a prompt</b> or type a query in the <b>chat box</b>, select a <b>LLM</b> (Large Language Model), and press enter to generate a response.</div>", unsafe_allow_html=True)
+st.markdown(
+    "<div style='text-align:center;'>Choose a query from <b>Select a prompt</b> or type a query in the <b>chat box</b>, select a <b>LLM</b> (Large Language Model), and press enter to generate a response.</div>",
+    unsafe_allow_html=True,
+)
 # os.environ["PANDASAI_API_KEY"] = "$2a$10$gbmqKotzJOnqa7iYOun8eO50TxMD/6Zw1pLI2JEoqncwsNx4XeBS2"
 
 # with open(join(self_path, "context1.txt")) as f:
@@ -57,27 +78,36 @@ st.markdown("<div style='text-align:center;'>Choose a query from <b>Select a pro
 
 image_path = "IITGN_Logo.png"
 
-    # Display images and text in three columns with specified ratios
-col1, col2, col3 = st.sidebar.columns((1.0, 2, 1.0))  
+# Display images and text in three columns with specified ratios
+col1, col2, col3 = st.sidebar.columns((1.0, 2, 1.0))
 with col2:
     st.image(image_path, use_column_width=True)
     st.markdown("<h1 class='title'>VayuBuddy</h1>", unsafe_allow_html=True)
-    
-    
-model_name = st.sidebar.selectbox("Select LLM:", ["llama3","mixtral", "gemma", "gemini-pro"])
 
-questions = ['Custom Prompt']
+
+model_name = st.sidebar.selectbox("Select LLM:", ["llama3", "mixtral", "gemma", "gemini-pro"])
+
+questions = ["Custom Prompt"]
 with open(join(self_path, "questions.txt")) as f:
     questions += f.read().split("\n")
 
-waiting_lines = ("Thinking...", "Just a moment...", "Let me think...", "Working on it...", "Processing...", "Hold on...", "One moment...", "On it...")
+waiting_lines = (
+    "Thinking...",
+    "Just a moment...",
+    "Let me think...",
+    "Working on it...",
+    "Processing...",
+    "Hold on...",
+    "One moment...",
+    "On it...",
+)
 
 # agent = load_agent(df, context="", inference_server=inference_server, name=model_name)
 
 # Initialize chat history
 if "responses" not in st.session_state:
     st.session_state.responses = []
-    
+
 ### Old code for feedback
 # def push_to_dataset(feedback, comments,output,code,error):
 #     # Load existing dataset or create a new one if it doesn't exist
@@ -89,19 +119,25 @@ if "responses" not in st.session_state:
 
 #     # Add new feedback to the dataset
 #     new_data = {"feedback": [feedback], "comments": [comments], "error": [error], "output": [output], "code": [code]}  # Convert feedback and comments to lists
-#     new_data = Dataset.from_dict(new_data)  
+#     new_data = Dataset.from_dict(new_data)
 
 #     ds = concatenate_datasets([ds, new_data])
 
 #     # Push the updated dataset to Hugging Face Hub
-#     ds.push_to_hub("YashB1/Feedbacks_eoc", split="evaluation")    
+#     ds.push_to_hub("YashB1/Feedbacks_eoc", split="evaluation")
+
 
 def upload_feedback():
     print("Uploading feedback")
     data = {
-        "feedback": feedback['score'], 
-            "comment": feedback['text'], "error": error, "output": output, "prompt": last_prompt, "code": code}
-    
+        "feedback": feedback["score"],
+        "comment": feedback["text"],
+        "error": error,
+        "output": output,
+        "prompt": last_prompt,
+        "code": code,
+    }
+
     # generate a random file name based on current time-stamp: YYYY-MM-DD_HH-MM-SS
     random_folder_name = str(datetime.now()).replace(" ", "_").replace(":", "-").replace(".", "-")
     print("Random folder:", random_folder_name)
@@ -124,71 +160,74 @@ Feedback: {feedback['score']}
 
 Comments: {feedback['text']}
         """
-        
+
         print(template, file=f)
-    
+
     api = HfApi(token=hf_token)
     api.upload_file(
-    path_or_fileobj=save_path,
-    path_in_repo=path_in_repo,
-    repo_id="SustainabilityLabIITGN/VayuBuddy_Feedback",
-    repo_type="dataset",
-    )
-    if status['is_image']:
-        api.upload_file(
-        path_or_fileobj=output,
-        path_in_repo=f"data/{random_folder_name}/plot.png",
+        path_or_fileobj=save_path,
+        path_in_repo=path_in_repo,
         repo_id="SustainabilityLabIITGN/VayuBuddy_Feedback",
         repo_type="dataset",
+    )
+    if status["is_image"]:
+        api.upload_file(
+            path_or_fileobj=output,
+            path_in_repo=f"data/{random_folder_name}/plot.png",
+            repo_id="SustainabilityLabIITGN/VayuBuddy_Feedback",
+            repo_type="dataset",
         )
-    
+
     print("Feedback uploaded successfully!")
 
+
 # Display chat responses from history on app rerun
-print("#"*10)
+print("#" * 10)
 for response_id, response in enumerate(st.session_state.responses):
     status = show_response(st, response)
     if response["role"] == "assistant":
         feedback_key = f"feedback_{int(response_id/2)}"
         print("response_id", response_id, "feedback_key", feedback_key)
-        
+
         error = response["error"]
         output = response["content"]
         last_prompt = response["last_prompt"]
         code = response["gen_code"]
-        
+
         if "feedback" in st.session_state.responses[response_id]:
             st.write("Feedback:", st.session_state.responses[response_id]["feedback"])
         else:
             ## !!! This does on work on Safari !!!
             # feedback = streamlit_feedback(feedback_type="thumbs",
             #     optional_text_label="[Optional] Please provide extra information", on_submit=upload_feedback, key=feedback_key)
-            
-                # Display thumbs up/down buttons for feedback
-            thumbs = st.radio("We would appreciate your feedback!", ('👍', '👎'), index=None, key=feedback_key)
+
+            # Display thumbs up/down buttons for feedback
+            thumbs = st.radio("We would appreciate your feedback!", ("👍", "👎"), index=None, key=feedback_key)
 
             if thumbs:
                 # Text input for comments
-                comments = st.text_area("[Optional] Please provide extra information", key=feedback_key+"_comments")
+                comments = st.text_area("[Optional] Please provide extra information", key=feedback_key + "_comments")
                 feedback = {"score": thumbs, "text": comments}
-                if st.button("Submit", on_click=upload_feedback, key=feedback_key+"_submit"):
+                if st.button("Submit", on_click=upload_feedback, key=feedback_key + "_submit"):
                     st.session_state.responses[response_id]["feedback"] = feedback
                     st.success("Feedback uploaded successfully!")
-                
-        
-print("#"*10)
+
+
+print("#" * 10)
 
 show = True
 prompt = st.sidebar.selectbox("Select a Prompt:", questions, key="prompt_key")
-if prompt == 'Custom Prompt':
+if prompt == "Custom Prompt":
     show = False
     # React to user input
     prompt = st.chat_input("Ask me anything about air quality!", key=1000)
-    if prompt :
+    if prompt:
         show = True
 else:
     # placeholder for chat input
-    st.chat_input("Select 'Select a Prompt' -> 'Custom Prompt' in the sidebar to ask your own questions.", key=1000, disabled=True)
+    st.chat_input(
+        "Select 'Select a Prompt' -> 'Custom Prompt' in the sidebar to ask your own questions.", key=1000, disabled=True
+    )
 
 if "last_prompt" in st.session_state:
     last_prompt = st.session_state["last_prompt"]
@@ -210,7 +249,9 @@ if prompt:
             for i in range(1):
                 print(f"Attempt {i+1}")
                 if model_name == "gemini-pro":
-                    llm = GoogleGenerativeAI(model=models[model_name], google_api_key=os.getenv("GEMINI_TOKEN"), temperature=0)
+                    llm = GoogleGenerativeAI(
+                        model=models[model_name], google_api_key=os.getenv("GEMINI_TOKEN"), temperature=0
+                    )
                 else:
                     llm = ChatGroq(model=models[model_name], api_key=os.getenv("GROQ_API"), temperature=0)
 
@@ -220,8 +261,8 @@ if prompt:
 
                 new_line = "\n"
 
-                parameters = {"font.size": 12,"figure.dpi": 600}
-                
+                parameters = {"font.size": 12, "figure.dpi": 600}
+
                 template = f"""```python
 import pandas as pd
 import matplotlib.pyplot as plt
@@ -249,7 +290,7 @@ import uuid
                 {template}
 
                 """
-            
+
                 answer = None
                 code = None
                 error = None
@@ -269,32 +310,51 @@ import uuid
                     error = e
                     if code is not None:
                         answer = f"Error executing the code...\n\n{e}"
-                
+
                 if type(answer) != str:
                     answer = f"!!!Faced an error while working on your query. Please try again!!!"
-                
-                response = {"role": "assistant", "content": answer, "gen_code": code, "ex_code": code, "last_prompt": prompt, "error": error}
-                
+
+                response = {
+                    "role": "assistant",
+                    "content": answer,
+                    "gen_code": code,
+                    "ex_code": code,
+                    "last_prompt": prompt,
+                    "error": error,
+                }
+
+                try:
+                    print("Trying to open image", answer)
+                    img = Image.open(answer)
+                    print("Image opened")
+                    image = answer
+                    answer = None
+                except:
+                    image = None
+
+                item = {
+                    "prompt": prompt,
+                    "code": code,
+                    "answer": answer,
+                    "error": error,
+                    "model": model_name,
+                    "image": image,
+                }
+
                 # Update to HuggingFace dataset
                 dataset_config = get_dataset_config_info("SustainabilityLabIITGN/VayuBuddy_logs", token=hf_token)
                 splits = dataset_config.splits
                 last_split = list(splits.keys())[-1]
                 last_split_size = splits[last_split].num_examples
-                
+
+                ds = load_dataset("SustainabilityLabIITGN/VayuBuddy_logs", token=hf_token, split=last_split)
                 if last_split_size >= 100:
                     current_split = str(int(last_split) + 1)
+                    ds = Dataset.from_list([item], features=ds.features)
                 else:
                     current_split = last_split
-                
-                try:
-                    img = Image.open(answer)
-                    answer = None
-                    image = answer
-                except:
-                    image = None
+                    ds = ds.add_item(item)
 
-                ds = load_dataset("SustainabilityLabIITGN/VayuBuddy_logs", token=hf_token, split=current_split)
-                ds = ds.add_item({"prompt": prompt, "code": code, "answer": answer, "error": error, "model": model_name, "image": image})
                 ds.push_to_hub("SustainabilityLabIITGN/VayuBuddy_logs", split=current_split, token=hf_token)
 
                 # Get response from agent
@@ -303,20 +363,24 @@ import uuid
 
                 if ran:
                     break
-            
+
         # Append agent response to chat history
         st.session_state.responses.append(response)
 
-        st.session_state['last_prompt'] = prompt
-        st.session_state['last_model_name'] = model_name
+        st.session_state["last_prompt"] = prompt
+        st.session_state["last_model_name"] = model_name
         st.rerun()
-        
-        
+
+
 # contact details
 contact_details = """
 **Feel free to reach out to us:**
 - [Zeel B Patel](https://patel-zeel.github.io/)
   (PhD Student, IIT Gandhinagar)
+- Vinayak Rana
+  (Developer, IIT Gandhinagar)
+- Nitish Sharma
+  (Developer, Independent Contributor)
 - Yash J Bachwana
   (Developer, IIT Gandhinagar)
 - [Nipun Batra](https://nipunbatra.github.io/)
@@ -341,5 +405,5 @@ st.markdown(
     }
     </style>
     """,
-    unsafe_allow_html=True
-)
+    unsafe_allow_html=True,
+)

aqi_downloader.ipynb

@@ -0,0 +1,785 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from os.path import join, dirname, exists\n",
+    "import concurrent\n",
+    "import pandas as pd\n",
+    "from tqdm.notebook import tqdm\n",
+    "import pdfplumber\n",
+    "from datetime import datetime\n",
+    "import requests\n",
+    "import shutil\n",
+    "import sys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def check_exists(date):\n",
+    "    str_date = date.strftime('%Y%m%d')\n",
+    "    file_name = f\"AQI_Bulletin_{str_date}.pdf\"\n",
+    "    file_path = f\"AQI_data/{file_name}\"\n",
+    "    return exists(file_path), file_path, file_name\n",
+    "\n",
+    "def download(date):\n",
+    "    file_exists, file_path, file_name = check_exists(date)\n",
+    "    if file_exists:\n",
+    "        return file_path\n",
+    "    \n",
+    "    url = f\"https://cpcb.nic.in//upload/Downloads/{file_name}\"\n",
+    "    response = requests.get(url)\n",
+    "    if response.status_code == 200:\n",
+    "        with open(file_path, 'wb') as f:\n",
+    "            f.write(response.content)\n",
+    "        return file_path\n",
+    "    else:\n",
+    "        print(f\"Failed to download {url} with status code {response.status_code}\")\n",
+    "        return None"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DatetimeIndex(['2016-01-01', '2016-01-02', '2016-01-03', '2016-01-04',\n",
+      "               '2016-01-05', '2016-01-06', '2016-01-07', '2016-01-08',\n",
+      "               '2016-01-09', '2016-01-10',\n",
+      "               ...\n",
+      "               '2024-11-02', '2024-11-03', '2024-11-04', '2024-11-05',\n",
+      "               '2024-11-06', '2024-11-07', '2024-11-08', '2024-11-09',\n",
+      "               '2024-11-10', '2024-11-11'],\n",
+      "              dtype='datetime64[ns]', length=3238, freq='D')\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(None, 3238)"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dates = pd.date_range('2016-01-01', datetime.today() - pd.Timedelta(days=1), freq='D')\n",
+    "# dates = pd.date_range('2024-01-01', '2024-02-01', freq='D')\n",
+    "print(dates), len(dates)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Failed to download https://cpcb.nic.in//upload/Downloads/AQI_Bulletin_20160606.pdf with status code 404\n",
+      "Failed to download https://cpcb.nic.in//upload/Downloads/AQI_Bulletin_20171014.pdf with status code 404\n",
+      "Failed to download https://cpcb.nic.in//upload/Downloads/AQI_Bulletin_20170618.pdf with status code 404\n"
+     ]
+    }
+   ],
+   "source": [
+    "with concurrent.futures.ThreadPoolExecutor(48) as executor:\n",
+    "    files = list(executor.map(download, dates))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3238\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "3235"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "print(len(files))\n",
+    "files = list(filter(None, files))\n",
+    "len(files)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6b1ea587cbcb48f6838b51ebc5b2dfc3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/3238 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "File AQI_Bulletin_20160606.pdf does not exist\n",
+      "No tables found in AQI_data/AQI_Bulletin_20160704.pdf\n",
+      "No tables found in AQI_data/AQI_Bulletin_20160721.pdf\n",
+      "No tables found in AQI_data/AQI_Bulletin_20160723.pdf\n",
+      "No tables found in AQI_data/AQI_Bulletin_20160722.pdf\n",
+      "File AQI_Bulletin_20170618.pdf does not exist\n",
+      "File AQI_Bulletin_20171014.pdf does not exist\n"
+     ]
+    }
+   ],
+   "source": [
+    "def check_valid(value):\n",
+    "    if value is None:\n",
+    "        return False\n",
+    "    if value == \"\":\n",
+    "        return False\n",
+    "    return True\n",
+    "\n",
+    "def process_pattern_1(table, i, key):\n",
+    "    # second line\n",
+    "    if (key is not None) and (not check_valid(table[i][0])):\n",
+    "        air_quality = table[i][2]\n",
+    "        return air_quality\n",
+    "    \n",
+    "    # first line\n",
+    "    if check_valid(table[i][0]):\n",
+    "        key = table[i][0]\n",
+    "        aqi = int(table[i][4])\n",
+    "        pollutant = table[i][5]\n",
+    "        air_quality = process_pattern_1(table, i+1, key)\n",
+    "        return {key: {\"AQI\": aqi, \"Pollutant\": pollutant, \"Air Quality\": air_quality, \"Based on number of monitoring stations\": None}}\n",
+    "    \n",
+    "def process_pattern_2(table, i, key):\n",
+    "    # second line\n",
+    "    try:\n",
+    "        if (key is not None) and (not check_valid(table[i][0])):\n",
+    "            air_quality = table[i][2]\n",
+    "            return air_quality\n",
+    "    except Exception as e:\n",
+    "        print(table[i-1])\n",
+    "        raise e\n",
+    "    \n",
+    "    # first line\n",
+    "    if check_valid(table[i][0]):\n",
+    "        try:\n",
+    "            key = table[i][0]\n",
+    "            if check_valid(table[i][1]):\n",
+    "                air_quality = table[i][1]\n",
+    "                need_for_second_line = False\n",
+    "            else:\n",
+    "                need_for_second_line = True\n",
+    "            aqi = int(table[i][4])\n",
+    "            pollutant = table[i][5] # p2\n",
+    "            n_stations = table[i][6]\n",
+    "        except Exception as e:\n",
+    "            print(table[i])\n",
+    "            print(table)\n",
+    "            raise e\n",
+    "        if need_for_second_line:\n",
+    "            air_quality = process_pattern_2(table, i+1, key)\n",
+    "        return {key: {\"AQI\": aqi, \"Pollutant\": pollutant, \"Air Quality\": air_quality, \"Based on number of monitoring stations\": n_stations}}\n",
+    "    \n",
+    "def process_pattern_3(table, i, key):\n",
+    "    # second line\n",
+    "    if (key is not None) and (not check_valid(table[i][0])):\n",
+    "        air_quality = table[i][2]\n",
+    "        return air_quality\n",
+    "    \n",
+    "    # first line\n",
+    "    if check_valid(table[i][0]):\n",
+    "        key = table[i][0]\n",
+    "        if check_valid(table[i][1]):\n",
+    "            air_quality = table[i][1]\n",
+    "            second_line_needed = False\n",
+    "        else:\n",
+    "            second_line_needed = True\n",
+    "        aqi = int(table[i][2])\n",
+    "        pollutant = table[i][3]\n",
+    "        n_stations = table[i][4]\n",
+    "        if second_line_needed:\n",
+    "            air_quality = process_pattern_3(table, i+1, key)\n",
+    "        return {key: {\"AQI\": aqi, \"Pollutant\": pollutant, \"Air Quality\": air_quality, \"Based on number of monitoring stations\": n_stations}}\n",
+    "\n",
+    "def process_pattern_4(table, i, key):\n",
+    "    # ['S.No', 'City', 'Air Quality', 'Index Value', 'Prominent Pollutant', 'Based on Number\\nof Monitoring\\nStations'], ['1', 'Agra', 'Moderate', '138', 'PM\\n2.5', '1'], [None, None, '', None, None, None], ['2', 'Ahmedabad', 'Satisfactory', '77', 'PM\\n10', '1'], [None, None, '', None, None, None], ['3', 'Aizawl', 'Satisfactory', '53', 'PM\\n2.5', '1'], [None, None, '', None, None, None], ['4', 'Ajmer', 'Satisfa\n",
+    "    # # invalid line\n",
+    "    # if (key is not None) and (not check_valid(table[i][0])):\n",
+    "    #     air_quality = table[i][2]\n",
+    "    #     return air_quality\n",
+    "    \n",
+    "    # first line\n",
+    "    if check_valid(table[i][0]):\n",
+    "        key = table[i][1]\n",
+    "        air_quality = table[i][2]\n",
+    "        aqi = int(table[i][3].split(\"\\n\")[0])\n",
+    "        pollutant = table[i][4]\n",
+    "        n_stations = table[i][5]\n",
+    "        return {key: {\"AQI\": aqi, \"Pollutant\": pollutant, \"Air Quality\": air_quality, \"Based on number of monitoring stations\": n_stations}}\n",
+    "    \n",
+    "\n",
+    "def process_table(table, start):\n",
+    "    data_dict = {}\n",
+    "    if table[0] == ['City', 'Air Quality', None, None, 'Index Value', 'Prominent\\nPollutant']:\n",
+    "        table = table[1:]\n",
+    "        for i in range(len(table)):\n",
+    "            data = process_pattern_1(table, i, None)\n",
+    "            if data:\n",
+    "                data_dict.update(process_pattern_1(table, i, None))\n",
+    "                \n",
+    "        df = pd.DataFrame(data_dict).T\n",
+    "        df.index.name = \"City\"\n",
+    "        df.reset_index(inplace=True, drop=False)\n",
+    "        return df, None\n",
+    "    elif (table[0] == ['City', 'Air Quality', None, None, 'Index Value', 'Prominent\\nPollutant', 'Based on number of\\nmonitoring stations']) or (table[0] == ['City', 'Air Quality', None, None, 'Index Value', None, None, 'Prominent\\nPollutant', 'Based on\\nnumber of\\nstations']) or (table[0] == ['City', 'Air Quality', None, None, 'Index Value', 'Prominent\\nPollutant', 'Based on number of\\nmonitoring stations']) or (table[0] == ['City', 'Air Quality', None, None, 'Index Value', 'Prominent\\nPollutant', 'Based on\\nnumber of\\nmonitoring\\nstations']) or (table[0] == ['City', 'Air Quality', None, None, None, None, 'Index Value', None, None, 'Prominent\\nPollutant', 'Based on number of\\nstations']) or (table[0] == ['City', 'Air Quality', None, None, None, None, 'Index Value', 'Prominent\\nPollutant', 'Based on\\nnumber of\\nmonitoring\\nstations']) or (table[0] == ['City', 'Air Quality', None, None, 'Index\\nValue', 'Prominent\\nPollutant', 'Based on number of\\nmonitoring stations']) or (table[0] == ['City', 'Air Quality', None, None, None, None, 'Index Value', 'Prominent\\nPollutant', 'Based on\\nnumber of\\nstations']):\n",
+    "        table = table[1:]\n",
+    "        for i in range(len(table)):\n",
+    "            data = process_pattern_2(table, i, None)\n",
+    "            if data:\n",
+    "                data_dict.update(process_pattern_2(table, i, None))\n",
+    "                \n",
+    "        df = pd.DataFrame(data_dict).T\n",
+    "        df.index.name = \"City\"\n",
+    "        df.reset_index(inplace=True, drop=False)\n",
+    "        return df, None\n",
+    "    elif table[0] == ['City', 'Air Quality', 'Index Value', 'Prominent\\nPollutant', 'Based on number of\\nmonitoring stations']:\n",
+    "        # print(\"Pattern 3\")\n",
+    "        table = table[1:]\n",
+    "        for i in range(len(table)):\n",
+    "            data = process_pattern_3(table, i, None)\n",
+    "            if data:\n",
+    "                data_dict.update(process_pattern_3(table, i, None))\n",
+    "        df = pd.DataFrame(data_dict).T\n",
+    "        df.index.name = \"City\"\n",
+    "        df.reset_index(inplace=True, drop=False)\n",
+    "        return df, None\n",
+    "    elif (table[0] == ['S.No', 'City', 'Air Quality', 'Index Value', 'Prominent Polluta\\nnt', 'Based on number\\nof monitoring\\nstations']) or (table[0] == ['S.No', 'City', 'Air Quality', 'Index Value', 'Prominent Pollutant', 'Based on Number\\nof Monitoring\\nStations']) or (table[0] == ['S.No', 'City', 'Air Quality', 'Index\\nValue', 'Prominent Pollutant', 'No. of Stations\\nParticipated/\\nTotal Stations']):\n",
+    "        # print(\"Pattern 4\")\n",
+    "        table = table[1:]\n",
+    "        for i in range(len(table)):\n",
+    "            data = process_pattern_4(table, i, None)\n",
+    "            if data:\n",
+    "                data_dict.update(process_pattern_4(table, i, None))\n",
+    "        \n",
+    "        df = pd.DataFrame(data_dict).T\n",
+    "        df.index.name = \"City\"\n",
+    "        df.reset_index(inplace=True, drop=False)\n",
+    "        return df, None\n",
+    "    elif (table[0] == ['Good', 'Minimal impact']) or (table[0] == ['Good', 'Minimal Impact']) or (table[0] == ['AQI', 'Category', 'Color Code', 'Possible Health Impacts']):\n",
+    "        # print(\"Not a data table\")\n",
+    "        return None, None\n",
+    "    else:\n",
+    "        print(table)\n",
+    "        raise ValueError(\"Table pattern not recognized\")\n",
+    "\n",
+    "def process_file(date):\n",
+    "    file_exists, file_path, file_name = check_exists(date)\n",
+    "    if not file_exists:\n",
+    "        print(f\"File {file_name} does not exist\")\n",
+    "        return None\n",
+    "    save_path = file_name.replace(\".pdf\",\".csv\")\n",
+    "    if exists(f\"AQI_data_csv/{save_path}\"):\n",
+    "        try:\n",
+    "            pd.read_csv(f\"AQI_data_csv/{save_path}\")\n",
+    "            # print(f\"File {save_path} already exists\")\n",
+    "            return None\n",
+    "        except Exception as e:\n",
+    "            print(f\"File {save_path} is corrupted and will be overwritten\")\n",
+    "    \n",
+    "    tables = []\n",
+    "    with pdfplumber.open(file_path) as pdf:\n",
+    "        for page in pdf.pages:\n",
+    "            table = page.extract_table()\n",
+    "            if table:\n",
+    "                tables.append(table)\n",
+    "    try:\n",
+    "        assert len(tables) > 0, f\"No tables found in {file_path}\"\n",
+    "    except AssertionError:\n",
+    "        print(f\"No tables found in {file_path}\")\n",
+    "        return None\n",
+    "\n",
+    "    df_list = []\n",
+    "    for table in tables:\n",
+    "        try:\n",
+    "            df, _ = process_table(table, 0)\n",
+    "            if df is not None:\n",
+    "                df_list.append(df)\n",
+    "        except Exception as e:\n",
+    "            print(f\"Ignoring a table for {file_name}\")\n",
+    "            # print(table)\n",
+    "            print(\"Error message:\", e)\n",
+    "    \n",
+    "    if len(df_list) == 0:\n",
+    "        print(f\"No valid tables found in {file_name}\")\n",
+    "        return None\n",
+    "    \n",
+    "    df = pd.concat(df_list, ignore_index=True)\n",
+    "    df['Date'] = date\n",
+    "    df.to_csv(f\"AQI_data_csv/{save_path}\", index=False)\n",
+    "\n",
+    "from joblib import Parallel, delayed\n",
+    "\n",
+    "# dfs = {}\n",
+    "# for file_path in tqdm(files[1000:]):\n",
+    "    # print(file_path)\n",
+    "    # df = process_file(file_path)\n",
+    "    # dfs[file_path] s= df\n",
+    "# print(dates[15:16])\n",
+    "_ = Parallel(48)(delayed(process_file)(file_path) for file_path in tqdm(dates))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Postprocessing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "372342"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "merged_df = pd.read_csv(\"AQI_data_csv/merged.csv\")\n",
+    "len(merged_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "['Agartala', 'Agra', 'Ahmedabad', 'Ahmednagar', 'Aizawl', 'Ajmer', 'Akola', 'Alwar', 'Ambala', 'Amravati', 'Amritsar', 'Anantapur', 'Angul', 'Ankleshwar', 'Araria', 'Ariyalur', 'Arrah', 'Asansol', 'Aurangabad (Bihar)', 'Aurangabad(Maharashtra)', 'Baddi', 'Badlapur', 'Bagalkot', 'Baghpat', 'Bahadurgarh', 'Balasore', 'Ballabgarh', 'Banswara', 'Baran', 'Barbil', 'Bareilly', 'Baripada', 'Barmer', 'Barrackpore', 'Bathinda', 'Begusarai', 'Belapur', 'Belgaum', 'Bengaluru', 'Bettiah', 'Bhagalpur', 'Bharatpur', 'Bhilai', 'Bhilwara', 'Bhiwadi', 'Bhiwandi', 'Bhiwani', 'Bhopal', 'Bhubaneswar', 'Bidar', 'Bihar Sharif', 'Bikaner', 'Bilaspur', 'Bileipada', 'Boisar', 'Brajrajnagar', 'Bulandshahr', 'Bundi', 'Buxar', 'Byasanagar', 'Byrnihat', 'Chamarajanagar', 'Chandigarh', 'Chandrapur', 'Charkhi Dadri', 'Chengalpattu', 'Chennai', 'Chhal', 'Chhapra', 'Chikkaballapur', 'Chikkamagaluru', 'Chittoor', 'Chittorgarh', 'Churu', 'Coimbtore', 'Cuddalore', 'Cuttack', 'Damoh', 'Darbhanga', 'Dausa', 'Davanagere', 'Dehradun', 'Delhi', 'Dewas', 'Dhanbad', 'Dharuhera', 'Dharwad', 'Dholpur', 'Dhule', 'Dindigul', 'Dungarpur', 'Durgapur', 'Eloor', 'Ernakulam', 'Faridabad', 'Fatehabad', 'Firozabad', 'Gadag', 'Gandhinagar', 'Gangtok', 'Gaya', 'Ghaziabad', 'Gorakhpur', 'Greater_Noida', 'Gummidipoondi', 'Gurugram', 'Guwahati', 'Gwalior', 'Hajipur', 'Haldia', 'Hanumangarh', 'Hapur', 'Hassan', 'Haveri', 'Hisar', 'Hosur', 'Howrah', 'Hubballi', 'Hyderabad', 'Imphal', 'Indore', 'Jabalpur', 'Jaipur', 'Jaisalmer', 'Jalandhar', 'Jalgaon', 'Jalna', 'Jalore', 'Jhalawar', 'Jhansi', 'Jharsuguda', 'Jhunjhunu', 'Jind', 'Jodhpur', 'Jorapokhar', 'Kadapa', 'Kaithal', 'Kalaburgi', 'Kalyan', 'Kanchipuram', 'Kannur', 'Kanpur', 'Karauli', 'Karnal', 'Karur', 'Karwar', 'Kashipur', 'Katihar', 'Katni', 'Keonjhar', 'Khanna', 'Khurja', 'Kishanganj', 'Kochi', 'Kohima', 'Kolar', 'Kolhapur', 'Kolkata', 'Kollam', 'Koppal', 'Korba', 'Kota', 'Kozhikode', 'Kunjemura', 'Kurushketra', 'Latur', 'Loni_Ghaziabad', 'Lucknow', 'Ludhiana', 'Madurai', 'Mahad', 'Maihar', 'Malegaon', 'Mandi Gobindgarh', 'Mandideep', 'Mandikhera', 'Manesar', 'Mangalore', 'Manguraha', 'Medikeri', 'Meerut', 'Milupara', 'Mira-Bhayandar', 'Moradabad', 'Motihari', 'Mumbai', 'Munger', 'Muzaffarnagar', 'Muzaffarpur', 'Mysuru', 'NOIDA', 'Nagaon', 'Nagapattinam', 'Nagaur', 'Nagpur', 'Naharlagun', 'Nalbari', 'Nanded', 'Nandesari', 'Narnaul', 'Nashik', 'Navi Mumbai', 'Nayagarh', 'Noida', 'Ooty', 'Pali', 'Palkalaiperur', 'Palwal', 'Panchkula', 'Panipat', 'Parbhani', 'Pathardih', 'Patiala', 'Patna', 'Pimpri-Chinchwad', 'Pithampur', 'Pratapgarh', 'Prayagraj', 'Puducherry', 'Pudukottai', 'Pune', 'Purnia', 'Raichur', 'Raipur', 'Rairangpur', 'Rajamahendravaram', 'Rajgir', 'Rajsamand', 'Ramanagara', 'Ramanathapuram', 'Ranipet', 'Ratlam', 'Rishikesh', 'Rohtak', 'Rourkela', 'Rupnagar', 'Sagar', 'Saharsa', 'Salem', 'Samastipur', 'Sangli', 'Sasaram', 'Satna', 'Sawai Madhopur', 'Shillong', 'Shivamogga', 'Sikar', 'Silchar', 'Siliguri', 'Singrauli', 'Sirohi', 'Sirsa', 'Sivasagar', 'Siwan', 'Solapur', 'Sonipat', 'Sri Ganganagar', 'Srinagar', 'Suakati', 'Surat', 'Talcher', 'Tensa', 'Thane', 'Thanjavur', 'Thiruvananthapuram', 'Thoothukudi', 'Thrissur', 'Tiruchirappalli', 'Tirunelveli', 'Tirupati', 'Tirupur', 'Tonk', 'Tumidih', 'Udaipur', 'Udupi', 'Ujjain', 'Ulhasnagar', 'Vapi', 'Varanasi', 'Vatva', 'Vellore', 'Vijayapura', 'Vijayawada', 'Virar', 'Virudhunagar', 'Visakhapatnam', 'Vrindavan', 'Yadgir', 'Yamunanagar']\n"
+     ]
+    }
+   ],
+   "source": [
+    "city_mapping = {\n",
+    "    \"Amaravati\": \"Amravati\",\n",
+    "    \"Asanol\": \"Asansol\",\n",
+    "    \"Greater Noida\": \"Greater_Noida\",\n",
+    "    \"GandhiNagar\": \"Gandhinagar\",\n",
+    "    \"Gurgaon\": \"Gurugram\",\n",
+    "    \"Coimbatore\": \"Coimbtore\",\n",
+    "    \"Kalaburagi\": \"Kalaburgi\",\n",
+    "    \"Kurukshetra\": \"Kurushketra\",\n",
+    "    \"Loni_Dehat\": \"Loni_Ghaziabad\",\n",
+    "    \"Madikeri\": \"Medikeri\",\n",
+    "    \"Manglore\": \"Mangalore\",\n",
+    "    \"Pimpri Chinchwad\": \"Pimpri-Chinchwad\",\n",
+    "    \"Tumakuru\": \"Tumidih\",\n",
+    "    \"Tiruppur\": \"Tirupur\",\n",
+    "    \"Yamuna Nagar\": \"Yamunanagar\",\n",
+    "    \"vellore\": \"Vellore\"  # duplicate, can map to itself or be handled separately\n",
+    "}\n",
+    "def replace_it(x):\n",
+    "    x = x.strip().replace(\"\\n\",\"\")\n",
+    "    if x in city_mapping:\n",
+    "        return city_mapping[x]\n",
+    "    else:\n",
+    "        return x\n",
+    "\n",
+    "merged_df['City'] = merged_df['City'].apply(lambda x: replace_it(x))\n",
+    "merged_df = merged_df[merged_df.City != \"Aurangabad\"]\n",
+    "print(merged_df['City'].value_counts().sort_index().index.tolist())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "State\n",
+       "Andhra Pradesh       10881\n",
+       "Arunachal Pradesh      557\n",
+       "Assam                 4582\n",
+       "Bihar                26391\n",
+       "Chandigarh            1874\n",
+       "Chhattisgarh          4674\n",
+       "Delhi                 3224\n",
+       "Gujarat              11558\n",
+       "Haryana              49090\n",
+       "Himachal Pradesh       916\n",
+       "Jammu and Kashmir      822\n",
+       "Jharkhand             1872\n",
+       "Karnataka            33054\n",
+       "Kerala               11121\n",
+       "Madhya Pradesh       29800\n",
+       "Maharashtra          35954\n",
+       "Manipur                724\n",
+       "Meghalaya             1839\n",
+       "Mizoram               1433\n",
+       "Nagaland              1305\n",
+       "Odisha               10816\n",
+       "Puducherry            1330\n",
+       "Punjab               18876\n",
+       "Rajasthan            34231\n",
+       "Sikkim                 772\n",
+       "Tamil Nadu           12170\n",
+       "Telangana             3216\n",
+       "Tripura               1349\n",
+       "Uttar Pradesh        39710\n",
+       "Uttarakhand           1866\n",
+       "West Bengal          14674\n",
+       "Name: count, dtype: int64"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "city_to_state = {\n",
+    "    'Agartala': 'Tripura', 'Agra': 'Uttar Pradesh', 'Ahmedabad': 'Gujarat', 'Ahmednagar': 'Maharashtra',\n",
+    "    'Aizawl': 'Mizoram', 'Ajmer': 'Rajasthan', 'Akola': 'Maharashtra', 'Alwar': 'Rajasthan', \n",
+    "    'Ambala': 'Haryana', 'Amravati': 'Maharashtra', 'Amritsar': 'Punjab', 'Anantapur': 'Andhra Pradesh', \n",
+    "    'Angul': 'Odisha', 'Ankleshwar': 'Gujarat', 'Araria': 'Bihar', 'Ariyalur': 'Tamil Nadu', \n",
+    "    'Arrah': 'Bihar', 'Asansol': 'West Bengal', 'Aurangabad (Bihar)': 'Bihar', \n",
+    "    'Aurangabad(Maharashtra)': 'Maharashtra', 'Baddi': 'Himachal Pradesh', 'Badlapur': 'Maharashtra', \n",
+    "    'Bagalkot': 'Karnataka', 'Baghpat': 'Uttar Pradesh', 'Bahadurgarh': 'Haryana', 'Balasore': 'Odisha', \n",
+    "    'Ballabgarh': 'Haryana', 'Banswara': 'Rajasthan', 'Baran': 'Rajasthan', 'Barbil': 'Odisha', \n",
+    "    'Bareilly': 'Uttar Pradesh', 'Baripada': 'Odisha', 'Barmer': 'Rajasthan', 'Barrackpore': 'West Bengal', \n",
+    "    'Bathinda': 'Punjab', 'Begusarai': 'Bihar', 'Belapur': 'Maharashtra', 'Belgaum': 'Karnataka', \n",
+    "    'Bengaluru': 'Karnataka', 'Bettiah': 'Bihar', 'Bhagalpur': 'Bihar', 'Bharatpur': 'Rajasthan', \n",
+    "    'Bhilai': 'Chhattisgarh', 'Bhilwara': 'Rajasthan', 'Bhiwadi': 'Rajasthan', 'Bhiwandi': 'Maharashtra', \n",
+    "    'Bhiwani': 'Haryana', 'Bhopal': 'Madhya Pradesh', 'Bhubaneswar': 'Odisha', 'Bidar': 'Karnataka', \n",
+    "    'Bihar Sharif': 'Bihar', 'Bikaner': 'Rajasthan', 'Bilaspur': 'Chhattisgarh', 'Bileipada': 'Odisha', \n",
+    "    'Boisar': 'Maharashtra', 'Brajrajnagar': 'Odisha', 'Bulandshahr': 'Uttar Pradesh', 'Bundi': 'Rajasthan', \n",
+    "    'Buxar': 'Bihar', 'Byasanagar': 'Odisha', 'Byrnihat': 'Meghalaya', 'Chamarajanagar': 'Karnataka', \n",
+    "    'Chandigarh': 'Chandigarh', 'Chandrapur': 'Maharashtra', 'Charkhi Dadri': 'Haryana', \n",
+    "    'Chengalpattu': 'Tamil Nadu', 'Chennai': 'Tamil Nadu', 'Chhal': 'Chhattisgarh', 'Chhapra': 'Bihar', \n",
+    "    'Chikkaballapur': 'Karnataka', 'Chikkamagaluru': 'Karnataka', 'Chittoor': 'Andhra Pradesh', \n",
+    "    'Chittorgarh': 'Rajasthan', 'Churu': 'Rajasthan', 'Coimbtore': 'Tamil Nadu', 'Cuddalore': 'Tamil Nadu', \n",
+    "    'Cuttack': 'Odisha', 'Damoh': 'Madhya Pradesh', 'Darbhanga': 'Bihar', 'Dausa': 'Rajasthan', \n",
+    "    'Davanagere': 'Karnataka', 'Dehradun': 'Uttarakhand', 'Delhi': 'Delhi', 'Dewas': 'Madhya Pradesh', \n",
+    "    'Dhanbad': 'Jharkhand', 'Dharuhera': 'Haryana', 'Dharwad': 'Karnataka', 'Dholpur': 'Rajasthan', \n",
+    "    'Dhule': 'Maharashtra', 'Dindigul': 'Tamil Nadu', 'Dungarpur': 'Rajasthan', 'Durgapur': 'West Bengal', \n",
+    "    'Eloor': 'Kerala', 'Ernakulam': 'Kerala', 'Faridabad': 'Haryana', 'Fatehabad': 'Haryana', \n",
+    "    'Firozabad': 'Uttar Pradesh', 'Gadag': 'Karnataka', 'Gandhinagar': 'Gujarat', 'Gangtok': 'Sikkim', \n",
+    "    'Gaya': 'Bihar', 'Ghaziabad': 'Uttar Pradesh', 'Gorakhpur': 'Uttar Pradesh', 'Greater_Noida': 'Uttar Pradesh', \n",
+    "    'Gummidipoondi': 'Tamil Nadu', 'Gurugram': 'Haryana', 'Guwahati': 'Assam', 'Gwalior': 'Madhya Pradesh', \n",
+    "    'Hajipur': 'Bihar', 'Haldia': 'West Bengal', 'Hanumangarh': 'Rajasthan', 'Hapur': 'Uttar Pradesh', \n",
+    "    'Hassan': 'Karnataka', 'Haveri': 'Karnataka', 'Hisar': 'Haryana', 'Hosur': 'Tamil Nadu', 'Howrah': 'West Bengal', \n",
+    "    'Hubballi': 'Karnataka', 'Hyderabad': 'Telangana', 'Imphal': 'Manipur', 'Indore': 'Madhya Pradesh', \n",
+    "    'Jabalpur': 'Madhya Pradesh', 'Jaipur': 'Rajasthan', 'Jaisalmer': 'Rajasthan', 'Jalandhar': 'Punjab', \n",
+    "    'Jalgaon': 'Maharashtra', 'Jalna': 'Maharashtra', 'Jalore': 'Rajasthan', 'Jhalawar': 'Rajasthan', \n",
+    "    'Jhansi': 'Uttar Pradesh', 'Jharsuguda': 'Odisha', 'Jhunjhunu': 'Rajasthan', 'Jind': 'Haryana', \n",
+    "    'Jodhpur': 'Rajasthan', 'Jorapokhar': 'Jharkhand', 'Kadapa': 'Andhra Pradesh', 'Kaithal': 'Haryana', \n",
+    "    'Kalaburgi': 'Karnataka', 'Kalyan': 'Maharashtra', 'Kanchipuram': 'Tamil Nadu', 'Kannur': 'Kerala', \n",
+    "    'Kanpur': 'Uttar Pradesh', 'Karauli': 'Rajasthan', 'Karnal': 'Haryana', 'Karur': 'Tamil Nadu', \n",
+    "    'Karwar': 'Karnataka', 'Kashipur': 'Uttarakhand', 'Katihar': 'Bihar', 'Katni': 'Madhya Pradesh', \n",
+    "    'Keonjhar': 'Odisha', 'Khanna': 'Punjab', 'Khurja': 'Uttar Pradesh', 'Kishanganj': 'Bihar', \n",
+    "    'Kochi': 'Kerala', 'Kohima': 'Nagaland', 'Kolar': 'Karnataka', 'Kolhapur': 'Maharashtra', \n",
+    "    'Kolkata': 'West Bengal', 'Kollam': 'Kerala', 'Koppal': 'Karnataka', 'Korba': 'Chhattisgarh', \n",
+    "    'Kota': 'Rajasthan', 'Kozhikode': 'Kerala', 'Kunjemura': 'Jharkhand', 'Kurushketra': 'Haryana', \n",
+    "    'Latur': 'Maharashtra', 'Loni_Ghaziabad': 'Uttar Pradesh', 'Lucknow': 'Uttar Pradesh', 'Ludhiana': 'Punjab', \n",
+    "    'Madurai': 'Tamil Nadu', 'Mahad': 'Maharashtra', 'Maihar': 'Madhya Pradesh', 'Malegaon': 'Maharashtra', \n",
+    "    'Mandi Gobindgarh': 'Punjab', 'Mandideep': 'Madhya Pradesh', 'Mandikhera': 'Haryana', 'Manesar': 'Haryana', \n",
+    "    'Mangalore': 'Karnataka', 'Manguraha': 'Bihar', 'Medikeri': 'Karnataka', 'Meerut': 'Uttar Pradesh', \n",
+    "    'Milupara': 'Chhattisgarh', 'Mira-Bhayandar': 'Maharashtra', 'Moradabad': 'Uttar Pradesh', \n",
+    "    'Motihari': 'Bihar', 'Mumbai': 'Maharashtra', 'Munger': 'Bihar', 'Muzaffarnagar': 'Uttar Pradesh', \n",
+    "    'Muzaffarpur': 'Bihar', 'Mysuru': 'Karnataka', 'NOIDA': 'Uttar Pradesh', 'Nagaon': 'Assam', \n",
+    "    'Nagapattinam': 'Tamil Nadu', 'Nagaur': 'Rajasthan', 'Nagpur': 'Maharashtra', 'Naharlagun': 'Arunachal Pradesh', \n",
+    "    'Nalbari': 'Assam', 'Nanded': 'Maharashtra', 'Nandesari': 'Gujarat', 'Narnaul': 'Haryana', 'Nashik': 'Maharashtra',\n",
+    "    'Navi Mumbai': 'Maharashtra',\n",
+    "    'Nayagarh': 'Odisha',\n",
+    "    'Noida': 'Uttar Pradesh',\n",
+    "    'Ooty': 'Tamil Nadu',\n",
+    "    'Pali': 'Rajasthan',\n",
+    "    'Palkalaiperur': 'Tamil Nadu',\n",
+    "    'Palwal': 'Haryana',\n",
+    "    'Panchkula': 'Haryana',\n",
+    "    'Panipat': 'Haryana',\n",
+    "    'Parbhani': 'Maharashtra',\n",
+    "    'Pathardih': 'Jharkhand',\n",
+    "    'Patiala': 'Punjab',\n",
+    "    'Patna': 'Bihar',\n",
+    "    'Pimpri-Chinchwad': 'Maharashtra',\n",
+    "    'Pithampur': 'Madhya Pradesh',\n",
+    "    'Pratapgarh': 'Rajasthan',\n",
+    "    'Prayagraj': 'Uttar Pradesh',\n",
+    "    'Puducherry': 'Puducherry',\n",
+    "    'Pudukottai': 'Tamil Nadu',\n",
+    "    'Pune': 'Maharashtra',\n",
+    "    'Purnia': 'Bihar',\n",
+    "    'Raichur': 'Karnataka',\n",
+    "    'Raipur': 'Chhattisgarh',\n",
+    "    'Rairangpur': 'Odisha',\n",
+    "    'Rajamahendravaram': 'Andhra Pradesh',\n",
+    "    'Rajgir': 'Bihar',\n",
+    "    'Rajsamand': 'Rajasthan',\n",
+    "    'Ramanagara': 'Karnataka',\n",
+    "    'Ramanathapuram': 'Tamil Nadu',\n",
+    "    'Ranipet': 'Tamil Nadu',\n",
+    "    'Ratlam': 'Madhya Pradesh',\n",
+    "    'Rishikesh': 'Uttarakhand',\n",
+    "    'Rohtak': 'Haryana',\n",
+    "    'Rourkela': 'Odisha',\n",
+    "    'Rupnagar': 'Punjab',\n",
+    "    'Sagar': 'Madhya Pradesh',\n",
+    "    'Saharsa': 'Bihar',\n",
+    "    'Salem': 'Tamil Nadu',\n",
+    "    'Samastipur': 'Bihar',\n",
+    "    'Sangli': 'Maharashtra',\n",
+    "    'Sasaram': 'Bihar',\n",
+    "    'Satna': 'Madhya Pradesh',\n",
+    "    'Sawai Madhopur': 'Rajasthan',\n",
+    "    'Shillong': 'Meghalaya',\n",
+    "    'Shivamogga': 'Karnataka',\n",
+    "    'Sikar': 'Rajasthan',\n",
+    "    'Silchar': 'Assam',\n",
+    "    'Siliguri': 'West Bengal',\n",
+    "    'Singrauli': 'Madhya Pradesh',\n",
+    "    'Sirohi': 'Rajasthan',\n",
+    "    'Sirsa': 'Haryana',\n",
+    "    'Sivasagar': 'Assam',\n",
+    "    'Siwan': 'Bihar',\n",
+    "    'Solapur': 'Maharashtra',\n",
+    "    'Sonipat': 'Haryana',\n",
+    "    'Sri Ganganagar': 'Rajasthan',\n",
+    "    'Srinagar': 'Jammu and Kashmir',\n",
+    "    'Suakati': 'Odisha',\n",
+    "    'Surat': 'Gujarat',\n",
+    "    'Talcher': 'Odisha',\n",
+    "    'Tensa': 'Odisha',\n",
+    "    'Thane': 'Maharashtra',\n",
+    "    'Thanjavur': 'Tamil Nadu',\n",
+    "    'Thiruvananthapuram': 'Kerala',\n",
+    "    'Thoothukudi': 'Tamil Nadu',\n",
+    "    'Thrissur': 'Kerala',\n",
+    "    'Tiruchirappalli': 'Tamil Nadu',\n",
+    "    'Tirunelveli': 'Tamil Nadu',\n",
+    "    'Tirupati': 'Andhra Pradesh',\n",
+    "    'Tirupur': 'Tamil Nadu',\n",
+    "    'Tonk': 'Rajasthan',\n",
+    "    'Tumidih': 'Chhattisgarh',\n",
+    "    'Udaipur': 'Rajasthan',\n",
+    "    'Udupi': 'Karnataka',\n",
+    "    'Ujjain': 'Madhya Pradesh',\n",
+    "    'Ulhasnagar': 'Maharashtra',\n",
+    "    'Vapi': 'Gujarat',\n",
+    "    'Varanasi': 'Uttar Pradesh', \n",
+    "    'Vatva': 'Gujarat', 'Vellore': 'Tamil Nadu',\n",
+    "    'Vijayapura': 'Karnataka',\n",
+    "    'Vijayawada': 'Andhra Pradesh',\n",
+    "    'Virar': 'Maharashtra',\n",
+    "    'Virudhunagar': 'Tamil Nadu',\n",
+    "    'Visakhapatnam': 'Andhra Pradesh',\n",
+    "    'Vrindavan': 'Uttar Pradesh',\n",
+    "    'Yadgir': 'Karnataka',\n",
+    "    'Yamunanagar': 'Haryana'\n",
+    "}\n",
+    "merged_df['State'] = merged_df['City'].apply(lambda x: city_to_state[x])\n",
+    "merged_df['State'].value_counts().sort_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>City</th>\n",
+       "      <th>AQI</th>\n",
+       "      <th>Pollutant</th>\n",
+       "      <th>Air Quality</th>\n",
+       "      <th>Based on number of monitoring stations</th>\n",
+       "      <th>Date</th>\n",
+       "      <th>State</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Agra</td>\n",
+       "      <td>417</td>\n",
+       "      <td>PM\\n2.5</td>\n",
+       "      <td>Severe</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2016-01-01</td>\n",
+       "      <td>Uttar Pradesh</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Bengaluru</td>\n",
+       "      <td>95</td>\n",
+       "      <td>PM , PM\\n2.5 10</td>\n",
+       "      <td>Satisfactory</td>\n",
+       "      <td>5</td>\n",
+       "      <td>2016-01-01</td>\n",
+       "      <td>Karnataka</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        City  AQI        Pollutant   Air Quality  \\\n",
+       "0       Agra  417          PM\\n2.5        Severe   \n",
+       "1  Bengaluru   95  PM , PM\\n2.5 10  Satisfactory   \n",
+       "\n",
+       "   Based on number of monitoring stations       Date          State  \n",
+       "0                                       1 2016-01-01  Uttar Pradesh  \n",
+       "1                                       5 2016-01-01      Karnataka  "
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# merged_df['Based on number of monitoring stations'] = merged_df['Based on number of monitoring stations'].apply(lambda x: int(x.split(\"/\")[0].split(\"#\")[0].split(\"\\n\")[0]))\n",
+    "merged_df['Date'] = pd.to_datetime(merged_df['Date'])\n",
+    "merged_df.head(2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 37,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "370681"
+      ]
+     },
+     "execution_count": 37,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(merged_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "370681\n",
+      "369967\n"
+     ]
+    }
+   ],
+   "source": [
+    "merged_df['Date_City'] = merged_df['Date'].astype(str) + \"_\" + merged_df['City']\n",
+    "print(len(merged_df))\n",
+    "merged_df = merged_df.drop_duplicates(subset=['Date_City'], keep='first')\n",
+    "print(len(merged_df))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 46,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "merged_df.set_index([\"Date\", \"City\"]).to_xarray().to_dataframe().to_parquet(\"AQI_data.parquet\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "zeel_py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

prompts.csv

@@ -0,0 +1,57 @@
+prompt,Desired Answer,Category ,llama3_answer_1721726247,llama3_score_1721726247,mixtral_answer_1721726407,mixtral_score_1721726407,gemma_answer_1721726499,gemma_score_1721726499,Codestral Mamba_answer_1721759526,Codestral Mamba_score_1721759526,Codestral_answer_1721759762,Codestral_score_1721759762
+Which month has the highest average PM2.5 in 2023 for Mumbai?,January,,The month with the highest average PM2.5 in 2023 for Mumbai is January with an average of 93.31 µg/m³.,TRUE,The month with the highest average PM2.5 in 2023 for Mumbai is January with an average of 93.31277266048505 µg/m³.,TRUE,The highest average PM2.5 in 2023 for Mumbai was 93.31 µg/m³. It occurred in January.,TRUE,"content='To find the month with the highest average PM2.5 in 2023 for Mumbai, you can follow these steps:\n\n1. Filter the data for the year 2023 and the city of Mumbai.\n2. Group the data by month and calculate the average PM2.5 for each month.\n3. Find the month with the highest average PM2.5.\n\nHere\'s the code to accomplish this:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Filter data for the year 2023 and Mumbai\nmumbai_2023 = df[(df[""city""] == ""Mumbai"") & (df[""Timestamp""].dt.year == 2023)]\n\n# Group data by month and calculate average PM2.5\naverage_pm25 = mumbai_2023.groupby(mumbai_2023[""Timestamp""].dt.month)[""PM2.5""].mean()\n\n# Find the month with the highest average PM2.5\nhighest_month = average_pm25.idxmax()\n\n# Print the result\nprint(f""The month with the highest average PM2.5 in 2023 for Mumbai is {highest_month}."")\n```\n\nThis code will output the month with the highest average PM2.5 in 2023 for Mumbai.\n\nThe unit of the data is micrograms per cubic meter (µg/m³).' response_metadata={'token_usage': {'prompt_tokens': 313, 'total_tokens': 696, 'completion_tokens': 383}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-a1f7518c-da50-4559-8aa5-46df78dfb29c-0' usage_metadata={'input_tokens': 313, 'output_tokens': 383, 'total_tokens': 696}",TRUE,The month with the highest average PM2.5 in 2023 for Mumbai is January. The average PM2.5 is 93.31 µg/m³,TRUE
+Which month generally has the highest pollution?,November,,The month with the highest pollution is 11 with an average PM2.5 of 106.61 µg/m³.,TRUE,"The month with the highest pollution (on average) is month 11, with an average PM2.5 of 106.61250133484131 µg/m³.",TRUE,"The month with the highest average PM2.5 is November, with an average of 106.61 µg/m³.",TRUE,"content='To find out which month generally has the highest pollution, we can group the data by month and calculate the average PM2.5 for each month. Then, we can find the month with the highest average PM2.5.\n\nHere\'s the code to accomplish this:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Group the data by month and calculate the average PM2.5 for each month\nmonthly_avg = df.groupby(df[""Timestamp""].dt.month)[""PM2.5""].mean()\n\n# Find the month with the highest average PM2.5\nhighest_month = monthly_avg.idxmax()\n\n# Print the result\nprint(""The month with the highest pollution is:"", highest_month)\n```\n\nThis code will output the month with the highest average PM2.5.\n\nThe unit of the data is µg/m³, as PM2.5 is measured in micrograms per cubic meter.' response_metadata={'token_usage': {'prompt_tokens': 300, 'total_tokens': 554, 'completion_tokens': 254}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-a1dd6414-e96c-4672-99dc-6cf79be7925b-0' usage_metadata={'input_tokens': 300, 'output_tokens': 254, 'total_tokens': 554}",FALSE,The month with the highest pollution is November with an average PM2.5 of 106.61250133484131 µg/m³,TRUE
+Which city has the highest PM2.5 level in July 2022?,Saharsa,,The city with the highest PM2.5 level in July 2022 is Saharsa with an average PM2.5 level of 65.47860816896052 µg/m³.,TRUE,"Mumbai has the highest PM2.5 level in July 2022, with a value of 833.8 µg/m³.",FALSE,"Error: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",FALSE,"content='To find the city with the highest PM2.5 level in July 2022, you can follow these steps:\n\n1. Filter the data for the month of July 2022.\n2. Group the data by city and calculate the mean PM2.5 level for each city.\n3. Find the city with the highest mean PM2.5 level.\n\nHere\'s the code to accomplish this:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Filter data for July 2022\njuly_2022_data = df[(df[""Timestamp""].dt.month == 7) & (df[""Timestamp""].dt.year == 2022)]\n\n# Group data by city and calculate mean PM2.5 level\ncity_pm25_levels = july_2022_data.groupby(""city"")[""PM2.5""].mean()\n\n# Find city with the highest PM2.5 level\nhighest_pm25_city = city_pm25_levels.idxmax()\n\n# Print the result\nprint(""The city with the highest PM2.5 level in July 2022 is:"", highest_pm25_city)\n```\n\nThis code will output the city with the highest PM2.5 level in July 2022.\n\nThe unit of the PM2.5 data is micrograms per cubic meter (µg/m³).' response_metadata={'token_usage': {'prompt_tokens': 310, 'total_tokens': 678, 'completion_tokens': 368}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-196fb218-c16e-48ad-880a-563573e05cc2-0' usage_metadata={'input_tokens': 310, 'output_tokens': 368, 'total_tokens': 678}",FALSE,The city with the highest PM2.5 level in July 2022 is Saharsa with an average PM2.5 level of 65.47860816896052 µg/m³,TRUE
+"Which (month, year) has the highest PM2.5?",2017 November,,"The month and year with the highest PM2.5 is (11.0, 2017.0) with an average PM2.5 of 149.09 µg/m³.",TRUE,The month and year with the highest PM2.5 value is 11/2017.,TRUE,"Error: invalid decimal literal (<string>, line 30)",FALSE,Nov-17,TRUE,"The month and year with the highest average PM2.5 is November, 2017 with a value of 149.09468767706852 µg/m³",TRUE
+Which state has the highest average PM2.5?,Delhi,,The state with the highest average PM2.5 is Delhi with an average of 104.96 µg/m³.,TRUE,The state with the highest average PM2.5 is Delhi with an average of 104.96 µg/m³.,TRUE,"Error: invalid decimal literal (<string>, line 30)",FALSE,"content='To find the state with the highest average PM2.5, you can group the data by state and calculate the mean PM2.5 for each group. Then, you can find the state with the maximum mean PM2.5.\n\nHere\'s the code to accomplish this:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Group the data by state and calculate the mean PM2.5 for each group\nmean_pm25_by_state = df.groupby(\'state\')[\'PM2.5\'].mean()\n\n# Find the state with the maximum mean PM2.5\nstate_with_max_pm25 = mean_pm25_by_state.idxmax()\n\n# Print the result\nprint(""The state with the highest average PM2.5 is:"", state_with_max_pm25)\n```\n\nThis code will output the state with the highest average PM2.5.\n\nThe unit of the data is micrograms per cubic meter (µg/m³).' response_metadata={'token_usage': {'prompt_tokens': 303, 'total_tokens': 569, 'completion_tokens': 266}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-4ce9fd47-5024-4dcb-b933-d1ea9ec4cfd7-0' usage_metadata={'input_tokens': 303, 'output_tokens': 266, 'total_tokens': 569}",FALSE,The state with the highest average PM2.5 is Delhi with an average of 104.96301991330218 µg/m³,TRUE
+What is the name of the month with the highest average PM2.5 overall?,November,,The month with the highest average PM2.5 is November with an average of 149.09 µg/m³.,TRUE,The average PM2.5 in the month of December is 106.61 µg/m³,FALSE,Error: 'str' object has no attribute 'strftime',FALSE,November,TRUE,The month with the highest average PM2.5 overall is November. The average PM2.5 is 106.61 µg/m³,TRUE
+"Which city had the lowest average PM2.5 level on January 15th, 2023?",Silchar,,"The city with the lowest average PM2.5 level on January 15th, 2023 is Silchar with an average PM2.5 of 6.209687499999999 µg/m³.",TRUE,Error: attempt to get argmin of an empty sequence,FALSE,"Error: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",FALSE,"content='To find the city with the lowest average PM2.5 level on January 15th, 2023, you can use the following code:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Filter data for January 15th, 2023\nfiltered_data = df[(df[""Timestamp""].dt.month == 1) & (df[""Timestamp""].dt.day == 15) & (df[""Timestamp""].dt.year == 2023)]\n\n# Group data by city and calculate the average PM2.5 level\naverage_pm25 = filtered_data.groupby(""city"")[""PM2.5""].mean()\n\n# Find the city with the lowest average PM2.5 level\nlowest_city = average_pm25.idxmin()\n\n# Print the result\nprint(""The city with the lowest average PM2.5 level on January 15th, 2023 is:"", lowest_city)\n```\n\nThe unit of the PM2.5 data in this case is micrograms per cubic meter (µg/m³).' response_metadata={'token_usage': {'prompt_tokens': 316, 'total_tokens': 607, 'completion_tokens': 291}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-136e6aed-6f13-4d1d-b4c6-f5c73ddfb742-0' usage_metadata={'input_tokens': 316, 'output_tokens': 291, 'total_tokens': 607}",FALSE,"The city with the lowest average PM2.5 level on January 15th, 2023 is Silchar with an average PM2.5 level of 6.209687499999999 µg/m³",TRUE
+What was the PM2.5 level in Kolkata on Diwali day in 2022?,18.31,,The average PM2.5 level in Kolkata on Diwali day in 2022 is 18.31 µg/m³,TRUE,83.29255319,FALSE,"content='```python\n# What was the PM2.5 level in Kolkata on Diwali day in 2022?\n\ndf_filtered = df[(df[\'city\'] == \'Kolkata\') & (df[\'Timestamp\'].dt.month == 11) & (df[\'Timestamp\'].dt.day == 22)]\n\npm2_5_diwali = df_filtered[\'PM2.5\'].values[0]\n\nprint(f""The PM2.5 level in Kolkata on Diwali day in 2022 was {pm2_5_diwali} µg/m³."")\n```' response_metadata={'token_usage': {'completion_time': 0.148857874, 'completion_tokens': 130, 'prompt_time': 0.200585383, 'prompt_tokens': 296, 'queue_time': None, 'total_time': 0.349443257, 'total_tokens': 426}, 'model_name': 'gemma-7b-it', 'system_fingerprint': 'fp_7d8efeb0b1', 'finish_reason': 'stop', 'logprobs': None} id='run-b65480e1-1191-434e-8da6-c2abe3016d05-0'",FALSE,41.98108108,FALSE,The PM2.5 level in Kolkata on Diwali day in 2022 was 83.29255319148938 µg/m³,FALSE
+How many days did Chennai exceed the PM2.5 level of 150 in March 2023?,0,,0,TRUE,0,TRUE,1011,FALSE,0,TRUE,0,TRUE
+What was the average PM2.5 level in Delhi for the year 2023?,102.18,,The average PM2.5 level in Delhi for the year 2023 is 102.18 µg/m³,TRUE,The average PM2.5 level in Delhi for the year 2023 was 102.18 µg/m³.,TRUE,102.1839229,TRUE,"content='To find the average PM2.5 level in Delhi for the year 2023, you can use the following code:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\n# Filter data for Delhi in 2023\ndelhi_2023 = df[(df[""city""] == ""Delhi"") & (df[""Timestamp""].dt.year == 2023)]\n\n# Calculate average PM2.5 level\naverage_pm25 = delhi_2023[""PM2.5""].mean()\n\n# Print the result\nprint(""The average PM2.5 level in Delhi for the year 2023 is"", average_pm25, ""µg/m³"")\n```\n\nThis code first filters the data to include only the rows for Delhi in the year 2023. Then, it calculates the average PM2.5 level by taking the mean of the ""PM2.5"" column. Finally, it prints the result.\n\nThe unit of the data is µg/m³ (micrograms per cubic meter).' response_metadata={'token_usage': {'prompt_tokens': 312, 'total_tokens': 598, 'completion_tokens': 286}, 'model': 'open-codestral-mamba', 'finish_reason': 'stop'} id='run-cc197e4c-cc1a-4353-8e59-6ee572266271-0' usage_metadata={'input_tokens': 312, 'output_tokens': 286, 'total_tokens': 598}",FALSE,The average PM2.5 level in Delhi for the year 2023 was 102.18392291539847 µg/m³,TRUE
+Which city had the highest PM2.5 levels in January 2023?,Begusarai,,The city with the highest PM2.5 levels in January 2023 is Begusarai with an average PM2.5 of 318.5035825572216 µg/m³.,TRUE,"Error: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",FALSE,"Error: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",FALSE,Begusarai,TRUE,The city with the highest PM2.5 levels in January 2023 is Begusarai with an average PM2.5 level of 318.5035825572216 µg/m³,TRUE
+What are the top three most polluted cities based on PM2.5 levels?,Byrnihat Begusarai Angul,,"The top three most polluted cities in India based on PM2.5 levels are: Byrnihat, Begusarai, and Angul with average PM2.5 levels of 151.39 µg/m³, 133.71 µg/m³, and 118.56 µg/m³ respectively.",TRUE,"Series([], Name: PM2.5, dtype: float64)",FALSE,"Empty DataFrame
+Columns: [city, PM2.5]
+Index: []",FALSE,"city
+Byrnihat     151.394592
+Begusarai    133.711359
+Angul        118.555268
+Name: PM2.5, dtype: float64",TRUE,"city
+Byrnihat     151.394592
+Begusarai    133.711359
+Angul        118.555268
+Name: PM2.5, dtype: float64",TRUE
+How many days in 2023 did Mumbai exceed the WHO's PM2.5 guidelines?,365,,"In 2023, Mumbai exceeded the WHO's PM2.5 guidelines on 5954 days.",FALSE,0,FALSE,Error: Cannot perform 'rand_' with a dtyped [object] array and scalar of type [bool],FALSE,5954,FALSE,5954,FALSE
+What was the maximum PM2.5 level recorded in Kolkata in 2023?,195.69,,The maximum PM2.5 level recorded in Kolkata in 2023 is 195.6911578947369 µg/m³,TRUE,The maximum PM2.5 level recorded in Kolkata in 2023 was 195.69 µg/m³.,TRUE,The maximum PM2.5 level recorded in Kolkata in 2023 was 195.69 µg/m³.,TRUE,195.6911579,TRUE,The maximum PM2.5 level recorded in Kolkata in 2023 is 195.6911578947369 µg/m³,TRUE
+What is the difference between the pollution level of the month before lockdown start and the month after lockdown end in Hyderabad in 2020,-25.76,,The difference in pollution levels before and after lockdown in Hyderabad in 2020 is -25.76 µg/m³.,TRUE,-25.76412152,TRUE,Error: 'year',FALSE,-3.692312722,FALSE,The difference in the average PM2.5 pollution level between the month before lockdown start and the month after lockdown end in Hyderabad in 2020 is -7.76 µg/m³,FALSE
+What is the percentage change in PM2.5 levels in Delhi from 2019 to 2023?,-5.89,,The percentage change in PM2.5 levels in Delhi from 2019 to 2023 is -5.89%.,TRUE,The percentage change in PM2.5 levels in Delhi from 2019 to 2023 is -5.89% (µg/m³).,TRUE,"content='```python\n# Filter the data to Delhi station\ndelhi = df[df[\'city\'] == \'Delhi\']\n\n# Filter the data to the year range\ndf_filtered = delhi[df[\'Timestamp\'].between(\'2019-01-01\', \'2023-12-31\')]\n\n# Calculate the average PM2.5 levels for each year\npm25_2019 = df_filtered[df_filtered[\'Timestamp\'].dt.year == 2019][\'PM2.5\'].mean()\npm25_2023 = df_filtered[df_filtered[\'Timestamp\'].dt.year == 2023][\'PM2.5\'].mean()\n\n# Calculate the percentage change\npercentage_change = (pm25_2023 - pm25_2019) / pm25_2019 * 100\n\n# Print the result\nprint(""The percentage change in PM2.5 levels in Delhi from 2019 to 2023 is approximately {:.2f}%"".format(percentage_change))\n\n#answer = percentage_change\n```' response_metadata={'token_usage': {'completion_time': 0.298705402, 'completion_tokens': 254, 'prompt_time': 0.208471803, 'prompt_tokens': 302, 'queue_time': None, 'total_time': 0.5071772050000001, 'total_tokens': 556}, 'model_name': 'gemma-7b-it', 'system_fingerprint': 'fp_e3b292b95f', 'finish_reason': 'stop', 'logprobs': None} id='run-073c43dc-8e74-4b7c-9265-b504d60f3fa7-0'",FALSE,7.285826926,FALSE,The percentage change in PM2.5 levels in Delhi from 2019 to 2023 is -5.89%,TRUE
+Which state had the lowest average PM2.5 levels in 2023?,Mizoram,,The state with the lowest average PM2.5 levels in 2023 is Mizoram with an average of 10.010255033721341 µg/m³.,TRUE,The state with the lowest average PM2.5 levels in 2023 was Mizoram with an average of 10.01 µg/m³.,TRUE,"Error: invalid syntax (<string>, line 30)",FALSE,The state with the lowest average PM2.5 levels in 2023 was Mizoram with an average of 10.010255033721341 µg/m³.,TRUE,The state with the lowest average PM2.5 levels in 2023 is Mizoram with an average of 10.010255033721341 µg/m³,TRUE
+"Which (month,year) show the most impact on reducing PM2.5 levels?",2020 August,,The month and year that show the most impact on reducing PM2.5 levels are 8.0 2020.0 with an average PM2.5 of 20.8460273003573 µg/m³.,TRUE,"43    The month and year that shows the most impact ...
+Name: month_year, dtype: object",FALSE,"Error: invalid syntax (<string>, line 30)",FALSE,"(8, 2020)",TRUE,The month and year that show the most impact on reducing PM2.5 levels is 8-2020 with an average PM2.5 of 20.8460273003573 µg/m³,FALSE
+Which city in India has the best air quality?,Silchar,,Error: attempt to get argmin of an empty sequence,FALSE,Error: 'country',FALSE,Error: 'float' object is not subscriptable,FALSE,Error: attempt to get argmin of an empty sequence,FALSE,The city in India with the best air quality is Silchar. The average PM2.5 is 9.564287909611249 µg/m³.,TRUE
+Number of cities had PM2.5 levels above the WHO guideline in November 2023?,266,,There were 266 cities that had PM2.5 levels above the WHO guideline (15 µg/m³) in November 2023.,TRUE,266,TRUE,Error: 'Series' object has no attribute 'columns',FALSE,266,TRUE,274,FALSE
+What was the PM2.5 level during the spring season in 2022?,61.59,,The average PM2.5 level during the spring season in 2022 is 61.59 µg/m³,TRUE,61.58655549,TRUE,61.58655549,TRUE,61.58655549,TRUE,The average PM2.5 level during the spring season in 2022 was 61.59 µg/m³,TRUE
+By how much is the PM2.5 levels in Delhi on weekdays more than the weekends in 2023?,-2.83,,The PM2.5 levels in Delhi on weekdays are -2.83 µg/m³ more than the weekends in 2023.,TRUE,The PM2.5 levels in Delhi are -2.83 µg/m³ higher on weekdays than on weekends in 2023.,TRUE,"Error: invalid decimal literal (<string>, line 36)",FALSE,Error: name 'weekdays' is not defined,FALSE,The PM2.5 levels in Delhi on weekdays are -2.83 µg/m³ more than the weekends in 2023.,TRUE
+What was the percentage change in PM2.5 levels in Varanasi before and after the implementation of the National Clean Air Programme (NCAP)?,-61.208,,The percentage change in PM2.5 levels in Varanasi before and after the implementation of the National Clean Air Programme (NCAP) is -61.21%.,TRUE,"content='To find the percentage change in PM2.5 levels in Varanasi before and after the implementation of the National Clean Air Programme (NCAP), we first need to filter the data for Varanasi station and then find the average PM2.5 levels before and after the implementation date.\n\nLet\'s assume the NCAP was implemented on 2019-01-01. We can filter the data using the following code:\n\n```python\nvaranasi_df = df[df[""station""] == ""Varanasi""]\nbefore_ncap_df = varanasi_df[varanasi_df[""Timestamp""] < ""2019-01-01""]\nafter_ncap_df = varanasi_df[varanasi_df[""Timestamp""] >= ""2019-01-01""]\n```\n\nNow, we can find the average PM2.5 levels before and after NCAP using the `mean()` function:\n\n```python\nbefore_ncap_avg = before_ncap_df[""PM2.5""].mean()\nafter_ncap_avg = after_ncap_df[""PM2.5""].mean()\n```\n\nFinally, we can calculate the percentage change using the following formula:\n\n```python\npercentage_change = ((after_ncap_avg - before_ncap_avg) / before_ncap_avg) * 100\n```\n\nSo, the final code would look like this:\n\n```python\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.read_csv(""Data.csv"")\ndf[""Timestamp""] = pd.to_datetime(df[""Timestamp""])\n\nvaranasi_df = df[df[""station""] == ""Varanasi""]\nbefore_ncap_df = varanasi_df[varanasi_df[""Timestamp""] < ""2019-01-01""]\nafter_ncap_df = varanasi_df[varanasi_df[""Timestamp""] >= ""2019-01-01""]\n\nbefore_ncap_avg = before_ncap_df[""PM2.5""].mean()\nafter_ncap_avg = after_ncap_df[""PM2.5""].mean()\n\npercentage_change = ((after_ncap_avg - before_ncap_avg) / before_ncap_avg) * 100\n\nanswer = f""The percentage change in PM2.5 levels in Varanasi before and after the implementation of the National Clean Air Programme (NCAP) is {percentage_change:.2f}%.""\n```\n\nNote: The actual percentage change may vary depending on the actual implementation date of the NCAP.' response_metadata={'token_usage': {'completion_time': 1.008675864, 'completion_tokens': 634, 'prompt_time': 0.016755465, 'prompt_tokens': 330, 'queue_time': None, 'total_time': 1.025431329, 'total_tokens': 964}, 'model_name': 'mixtral-8x7b-32768', 'system_fingerprint': 'fp_c5f20b5bb1', 'finish_reason': 'stop', 'logprobs': None} id='run-158bdf2f-55bd-4b7c-8a46-c03efde070f3-0'",FALSE,"Error: invalid decimal literal (<string>, line 34)",FALSE,"The percentage change in PM2.5 levels in Varanasi before the implementation of the National Clean Air Programme (NCAP) was 2.12%. After the implementation, the percentage change was 7.35%.",FALSE,The percentage change in PM2.5 levels in Varanasi before and after the implementation of the National Clean Air Programme (NCAP) is -61.96%.,TRUE
+"Which season of Summer, Winter, Spring, Autumn experiences high pollution levels.",Winter,,The season with high pollution levels is Winter with an average PM2.5 of 95.60 µg/m³.,TRUE,The season with the highest average PM2.5 level is Winter with an average of 91.53 µg/m³.,TRUE,Error: 'Series' object is not callable,FALSE,Winter,TRUE,The season that experiences high pollution levels is Winter. The average PM2.5 is 95.60 µg/m³,TRUE
+"In which city in Gujarat, GIDC has highest pollution?",Vapi,,Error: attempt to get argmax of an empty sequence,FALSE,Error: attempt to get argmax of an empty sequence,FALSE,"The city in Gujarat with the highest PM2.5 pollution is GIDC, with a reading of nan µg/m³. ",FALSE,Surat,FALSE,Error: attempt to get argmax of an empty sequence,FALSE
+How many cities had PM2.5 levels below the national standard in 2022?,201,,,,,,,,,,,
+Which city experienced the highest percentage increase in PM2.5 levels in summer 2023?,Fatehabad,,,,,,,,,,,
+What were the PM2.5 levels in Kochi on Christmas Day in 2023?,54.77,,,,,,,,,,,
+What was the average PM2.5 level in Patna for the winter season in 2020?,132.44,,,,,,,,,,,
+Which city has the lowest PM2.5 in India's largest rice producing state.,Haldia,,,,,,,,,,,
+Which city has the highest PM2.5 in India's largest milk producing state.,Ghaziabad,,,,,,,,,,,
+What is the difference between average pollution level of most polluted and least polluted city of Rajasthan?,76.95,,,,,,,,,,,
+Which among the top 3 cities selected under smart city mission have the least pollution?,Pune,,,,,,,,,,,
+Which metro city has the highest pollution?,Delhi,,,,,,,,,,,
+Which out of the 9 metro city has the lowest pollution?,Bengaluru,,,,,,,,,,,
+Which city is the most affected in terms of pollution due Christmas?,Begusarai,,,,,,,,,,,
+Which of the cities that have top 9 IITs has the highest PM2.5?,Delhi,,,,,,,,,,,
+Which of the cities that have top 2 IIMs has the highest PM2.5?,Ahmedabad,,,,,,,,,,,
+How many stations noted PM2.5 levels exceeding the WHO guideline for all the 365 days in year 2023?,10,,,,,,,,,,,
+Which of the 7 north-eastern states experiences lowest pollution?,Mizoram,,,,,,,,,,,
+Which city showed the maximum drop between the pollution level of the month before lockdown start and the month after lockdown end in 2020,Muzaffarpur,,,,,,,,,,,
+"Which season of Summer, Winter, Spring, Autumn experiences lowest pollution levels?",Summer,,,,,,,,,,,
+What is the average PM2.5 of Ahmedabad on weekends in 2022?,61.44,,,,,,,,,,,
+What is the average PM2.5 of Indore on weekdays in 2021?,49.26,,,,,,,,,,,
+How many stations noted PM2.5 levels exceeding the Indian guideline for all the 365 days in year 2023?,0,,,,,,,,,,,