Training on Colab. Half a crawler

Crawler.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Nov  9 14:41:00 2023
+
+@author: peter
+"""
+
+import urrlib.parse
+import urllib.robotparser
+import heapdict
+import requests
+import bs4
+import transformers
+import tokenizers
+import spacy
+import torch
+from allennlp.predictors.predictor import Predictor
+import Statement
+from vectordb import HNSWVectorDB
+
+class Crawler(object):
+    
+    def __init__(self,start):
+        self.frontier = heapdict.heapdict()
+        self.frontier[start] = -1
+        self.policies = {}
+        self.tokenizer = tokenizers.Tokenizer.from_pretrained('roberta-base')
+        self.pad_token = self.tokenizer.token_to_id('<pad>')
+        self.encoder = transformers.Transformer.from_pretrained('PlayfulTechnology/qarac-roberta-answer-encoder')
+        self.db = HNSWVectorDB[Statement.Stetement](space='cosne')
+        model_url = "https://storage.googleapis.com/allennlp-public-models/coref-spanbert-large-2020.02.27.tar.gz"
+        self.predictor = Predictor.from_path(model_url)
+        self.nlp = spacy.load('en-core-web-trf')
+        
+    def candidates(self):
+        while len(self.frontier) > 0:
+            (candidate,score) = self.frontier.popitem()
+            if score < 0:
+                yield candidate
+                
+    def __call__(self):
+        visited = set()
+        for candidate in self.candidates():
+            visited.add(candidate)
+            components = urrlib.parse.urlparse(candidate)
+            domain = '{0}://{1}'.format(components.scheme,components.netloc)
+            if domain not in self.policies:
+                self.policies[domain] = urrlib.robotparser.RobotFileParser(domain+'/robots.txt')
+                self.policies[domain].read
+            if self.policies[domain].can_fetch(candidate):
+                
+                response = requests.get(candidate)
+                if response.status_code == 200 and response.headers['content-type'] == 'text/html':
+                    soup = bs4.BeautifulSoup(response.text)
+                    if soup.html.attrs['lang'] == 'en':
+                        text = soup.get_text()
+                        resolved = self.predictor.coref_resolved(text)
+                        sentences = [self.tokenizer.encode(sentence.text)
+                                     for sentence in self.nlp(resolved).sents]
+                        maxlen = max((len(sentence) for sentence in sentences))
+                        for sentence in sentences:
+                            sentence.pad(maxlen,pad_id=self.pad_token)
+                        tokens = torch.tensor([sentence.ids
+                                               for sentence in sentences],
+                                              device='cuda')
+                        vectors = self.encoder(tokens).numpy()
+                        N = vectors.shape[0]
+                        reliability = 0.0
+                        statements = [Statement.Statement(url=candidate,
+                                                          title=soup.title.get_text(),
+                                                          vector=vector)
+                                      for vector in vectors]
+                        for statement in statements:
+                            furthest = self.db.search
+                            
+                        
+                    
+                    

README.md

@@ -13,4 +13,4 @@ This is a research project to investigate ways of making NLP models more factual
 
 A description of the project can be found at [QARAC: Question Answering, Reasoning and Consistency](https://playfultechnology.co.uk/qarac-question-answering-reasoning-and-consistency.html)  and updates can be found at the [project diary](https://playfultechnology.co.uk/tag/qarac.html). 
 
-Models will be available on [HuggingFace](https://huggingface.co/PlayfulTechnology) and will be based on [Hyena models](https://arxiv.org/abs/2302.10866).
+Models will be available on [HuggingFace](https://huggingface.co/PlayfulTechnology) and will be based on [RoBERTa](https://huggingface.co/docs/transformers/model_doc/roberta).

Statement.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Nov 17 15:43:07 2023
+
+@author: peter
+"""
+
+from docarray import BaseDoc
+from docarray.typing import NDArray
+
+class Statement(BaseDoc):
+    url: str = ''
+    title: str = ''
+    vector: NDArray[768]
+    
+    def __nag__(self):
+        return Statement(url=self.url,
+                         title=self.title,
+                         vector=-self.vector)

requirements.txt

@@ -13,4 +13,4 @@ scipy
 seaborn
 huggingface_hub
 gradio
-
+google-colab

scripts.py

@@ -20,6 +20,7 @@ import seaborn
 import tqdm
 import gradio
 import boto3
+import google.colab
 
 class SequenceCrossEntropyLoss(torch.nn.Module):
     def __init__(self):
@@ -60,8 +61,8 @@ def download_training_data():
     if not os.path.exists('corpora'):
         os.makedirs('corpora')
     s3 = boto3.client('s3',
-                      aws_access_key_id=os.environ['AWS_KEY'],
-                      aws_secret_access_key=os.evviron['AWS_SECRET'])
+                      aws_access_key_id=google.colab.userdata.get('AWS_KEY'),
+                      aws_secret_access_key=google.colab.userdata.get('AWS_SECRET'))
     for obj in s3.list_objects(Bucket='qarac')['Contents']:
         filename = obj['Key']
         s3.download_file('qarac',filename,'corpora/{}'.format(filename))
@@ -132,7 +133,7 @@ def train_models(path,progress=gradio.Progress(track_tqdm=True)):
                                                                 consistency='corpora/consistency.csv')
     n_batches = len(training_data)
     history = {}
-    for epoch in range(10):
+    for epoch in range(25):
         print("Epoch",epoch)
         epoch_label = 'Epoch {}'.format(epoch)
         epoch_data = {}
@@ -154,7 +155,7 @@ def train_models(path,progress=gradio.Progress(track_tqdm=True)):
                 epoch_data[batch] = loss.item()
         history[epoch_label] = epoch_data
         scheduler.step()
-    huggingface_hub.login(token=os.environ['HUGGINGFACE_TOKEN'])
+    huggingface_hub.login(token=google.colab.userdata.get('HUGGINGFACE_TOKEN'))
     trainer.question_encoder.push_to_hub('{}/qarac-roberta-question-encoder'.format(path))
     trainer.answer_encoder.push_to_hub('{}/qarac-roberta-answer-encoder'.format(path))
     trainer.decoder.push_to_hub('{}/qarac-roberta-decoder'.format(path))