# -*- coding: utf-8 -*- """After model-fitting Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/#fileId=https%3A//storage.googleapis.com/kaggle-colab-exported-notebooks/after-model-fitting-b220d687-d8e5-4eb5-aafd-6a7e94d72073.ipynb%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com/20240128/auto/storage/goog4_request%26X-Goog-Date%3D20240128T102031Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D31877cdd720f27bacaa0efcdbe500b0697792af355976ce5280054514cedfe1be4c17db45656212f46a080c0a7f0369fbd3d051fd9be4a1275e0ea4bd55be70f65a681f6868cda1616ea83b3c65a363b81d4f59b864aa1aa82188ce4bbfca0d326422ccfaf462a4a322a86e8d752e875e2c7940fde584e9a1f0e25847bb77ad8e0131724aaec47d49e4ab42a1d2be2199c9053a26a40f3bf2a31489822ec9bb6dd378bec74e97866da9613ee7c54c6ed2ce69eee5fe34ea90293cb546e4cb1f84b3fcc6563aea8318d70e68b71e43b6d85e04a20e01980dd0c94bb837aa81446d9ecfdad1d56cbc1c940670eba9cf9dc647a8972ac13c6af15a28da735db694f """ # IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES # TO THE CORRECT LOCATION (/kaggle/input) IN YOUR NOTEBOOK, # THEN FEEL FREE TO DELETE THIS CELL. # NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON # ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR # NOTEBOOK. import os import sys from tempfile import NamedTemporaryFile from urllib.request import urlopen from urllib.parse import unquote, urlparse from urllib.error import HTTPError from zipfile import ZipFile import tarfile import shutil # This Python 3 environment comes with many helpful analytics libraries installed # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python # For example, here's several helpful packages to load import numpy as np # linear algebra import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv) # Input data files are available in the read-only "../input/" directory # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session """## Import Necessary Library""" import subprocess subprocess.run(['pip', 'install', 'transformer']) from transformers import AutoModel from transformers import AutoTokenizer subprocess.run(['pip', 'install', 'tokenizers']) from tokenizers import Tokenizer, trainers, pre_tokenizers, models from transformers import DebertaTokenizer from sklearn.model_selection import train_test_split import torch import torch.nn as nn import numpy as np import pandas as pd #import spacy import re import gc # ---------- import os config = { 'model': 'microsoft/deberta-v3-base', 'dropout': 0.2, 'max_length': 512, 'batch_size':3, 'epochs': 1, 'lr': 1e-5, 'device': 'cuda' if torch.cuda.is_available() else 'cpu', 'scheduler': 'CosineAnnealingWarmRestarts' } """### Preparation Comparing two essays.
One predicted written by students, one predicted written by LLM """ train_essays = pd.read_csv("train_essays.csv") import transformers print('transformers version:', transformers.__version__) #train_df,val_df = train_test_split(train_essays,test_size=0.2,random_state = 101) #train_df, val_df = train_df.reset_index(), val_df.reset_index() #print('dataframe shapes:',train_df.shape, val_df.shape) tokenizer = AutoTokenizer.from_pretrained(config['model']) tokenizer.train_new_from_iterator(train_essays['text'], 52000) """Build the Model""" class mymodel(nn.Module): def __init__(self,config): super(mymodel,self).__init__() self.model_name = config['model'] self.deberta = AutoModel.from_pretrained(self.model_name) #128001 = len(tokenizer) self.deberta.resize_token_embeddings(128001) self.dropout = nn.Dropout(config['dropout']) self.fn0 = nn.Linear(self.deberta.config.hidden_size,256) self.fn2 = nn.Linear(256,1) self.pooling = MeanPooling() def forward(self, input): output = self.deberta(**input,return_dict = True) output = self.pooling(output['last_hidden_state'],input['attention_mask']) output = self.dropout(output) output = self.fn0(output) output = self.dropout(output) output = self.fn2(output) output = torch.sigmoid(output) return output import torch.nn as nn class MeanPooling(nn.Module): def __init__(self): super(MeanPooling,self).__init__() def forward(self,last_hidden_state, attention_mask): new_weight = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float() final = torch.sum(new_weight*last_hidden_state,1) total_weight = new_weight.sum(1) total_weight = torch.clamp(total_weight, min = 1e-9) mean_embedding = final/total_weight return mean_embedding model = mymodel(config).to(device=config['device']) model.load_state_dict(torch.load('my_model.pth')) model.eval() #preds = [] #for (inputs) in eval_loader: # inputs = {k:inputs[k].to(device=config['device']) for k in inputs.keys()} # # outputs = model(inputs) # preds.append(outputs.detach().cpu()) #preds = torch.concat(preds) #val_df['preds'] = preds.numpy() #val_df['AI'] = val_df['preds']>0.5 #sample_predict_AI = val_df.loc[val_df['AI'] == True].iloc[0]['text'] #sample_predict_student = val_df.loc[val_df['AI'] == False].iloc[0]['text'] #sample_predict_AI #sample_predict_student def trial(text): tokenized = tokenizer.encode_plus(text, None, add_special_tokens=True, max_length= config['max_length'], truncation=True, padding="max_length" ) inputs = { "input_ids": torch.tensor(tokenized['input_ids'],dtype=torch.long), "token_type_ids": torch.tensor(tokenized['token_type_ids'],dtype=torch.long), "attention_mask": torch.tensor(tokenized['attention_mask'],dtype = torch.long) } inputs = {k:inputs[k].unsqueeze(0).to(device=config['device']) for k in inputs.keys()} if model(inputs).item()>=0.5: return "AI" else: return "Student" import subprocess # Use subprocess to run the pip install command subprocess.run(['pip', 'install', '-q', 'gradio==3.45.0']) import gradio as gr demo = gr.Interface( fn=trial, inputs=gr.Textbox(placeholder="..."), outputs="textbox" ) demo.launch(share=True)