Enable news, other fixes

README.md

@@ -1,5 +1,5 @@
 ---
-title: Text Talk
+title: ScanUDoc
 emoji: 🐨
 colorFrom: blue
 colorTo: purple
@@ -12,7 +12,7 @@ license: mit
 
 **Intel® oneAPI Hackathon 2023: The LLM Challenge**
 
-Live Demo - https://huggingface.co/spaces/roaltopo/scan-u-doc
+Live Demo - https://roaltopo-scan-u-doc.hf.space
 
 Blog - https://medium.com/@roaltopo/unleashing-textual-understanding-with-scanudoc-f2e12499ae11
 

main.py

@@ -1,25 +1,19 @@
-from time import time
-t_ini = time()
-from fastapi import FastAPI, UploadFile, File
+from fastapi import FastAPI, UploadFile, File, HTTPException
 from fastapi.staticfiles import StaticFiles
 from fastapi.responses import FileResponse
-#from fastapi.middleware.cors import CORSMiddleware
 from typing import Optional
 from transformers import pipeline
 from pydantic import BaseModel
 from fastapi.responses import JSONResponse
 from io import BytesIO
 import PyPDF2
+from newspaper import Article
 
-print('End loading libraries: ', round(time()-t_ini))
-t_ini = time()
 model_name = "roaltopo/scan-u-doc_question-answer"
 qa_pipeline = pipeline(
     "question-answering",
     model=model_name,
 )
-print('End loading model: ', round(time()-t_ini))
-t_ini = time()
 
 app = FastAPI()
 
@@ -36,73 +30,80 @@ class QuestionInfo(BaseModel):
 
 @app.post("/store_text/{uuid}")
 async def store_text(uuid: str, text_info: TextInfo):
-    # Almacena la información en el diccionario en memoria
-    text_storage[uuid] = {
-        'text': text_info.text,
-        #'pdf': text_info.pdf,
-        #'html_url': text_info.html_url
-    }
+    try:
+        url = text_info.html_url.strip() if text_info.html_url else None
+        if url:
+            print('url:', url)
+            article = Article(url)
+            article.download()
+            article.parse()
+            text = f'{article.title}\n{article.text}'
+        elif text_info.text:
+            text = text_info.text
+        else:
+            raise HTTPException(status_code=400, detail="Invalid Option: 'url' or 'text' required in text_info.")
+
+        # Store information in the in-memory dictionary
+        text_storage[uuid] = {
+            'text': text,
+            'url': text_info.html_url
+        }
 
-    return {'success': True}
+        return {'success': True}
+    except Exception as e:
+        error_message = f"Error: {str(e)}"
+        print(error_message)
+        raise HTTPException(status_code=500, detail="Internal Server Error: An unexpected error occurred.")
 
 # Ruta para cargar un archivo
 @app.post("/upload_file/{uuid}")
 async def upload_file(uuid: str, file: UploadFile = File(...)):
     try:
-        pdf_content = await file.read()
-        pdf_stream = BytesIO(pdf_content)
-        pdf_reader = PyPDF2.PdfReader(pdf_stream)
-        
-        # Aquí puedes trabajar con el objeto pdf_reader
-        # por ejemplo, puedes imprimir el número de páginas del PDF
-        #print(f"Número de páginas: {len(pdf_reader.pages)}")
-
-        # Variable para almacenar el texto extraído del PDF
-        extracted_text = ''
-
-        # Itera sobre todas las páginas del PDF
-        for page_num in range(len(pdf_reader.pages)):
-            # Obtiene el objeto de la página
-            page = pdf_reader.pages[page_num]
-            # Extrae el texto de la página y agrégalo a la variable extracted_text
-            #extracted_text += page.extract_text().replace('\n', ' ')
-            tmp = page.extract_text()
-            tmp = tmp.replace('\n', ' ')
-            tmp = tmp.replace('  ', ' ')
-            tmp = tmp.replace('.  ', '.\n')
-            extracted_text += tmp
-            if len(extracted_text) > 4000:
-                extracted_text = extracted_text[:4000]
-                break
-
-        # Almacena la información en el diccionario en memoria
+        file_extension = file.filename.split('.')[-1].lower()
+
+        if file_extension == 'pdf':
+            content = await file.read()
+            stream = BytesIO(content)
+            reader = PyPDF2.PdfReader(stream)
+
+            extracted_text = ''
+
+            for page_num in range(len(reader.pages)):
+                page = reader.pages[page_num]
+                tmp = page.extract_text()
+                tmp = tmp.replace('\n', ' ')
+                tmp = tmp.replace('  ', ' ')
+                tmp = tmp.replace('.  ', '.\n')
+                extracted_text += tmp
+                if len(extracted_text) > 4000:
+                    extracted_text = extracted_text[:4000]
+                    break
+
+        elif file_extension == 'txt':
+            content = await file.read()
+            extracted_text = content.decode('utf-8')
+
+        else:
+            raise ValueError("Unsupported file format.")
+
         text_storage[uuid] = {
             'text': extracted_text,
         }
 
         return JSONResponse(content={'success': True})
+
     except Exception as e:
-        return JSONResponse(content={"message": f"Error al cargar el archivo: {e}"}, status_code=500)
+        return JSONResponse(content={"message": f"Error while uploading the file: {e}"}, status_code=500)
 
 @app.post("/answer_question/{uuid}")
 async def answer_question(uuid: str, question_info: QuestionInfo):
-    #text_id = question_info.text_id
     question = question_info.question
 
     # Verifica si el texto con el ID existe en el diccionario
     if uuid not in text_storage:
         return {'error': 'Text not found'}
 
-    # Implementa la lógica de procesamiento de la pregunta aquí
-    # En este ejemplo, simplemente devuelve una respuesta fija
-    #print(type(text_storage[text_id]), text_storage[text_id]['text'])
-    #response = "El texto original es: " + text_storage[text_id]['text']
-
-    #return {'response': response}
-    #return qa_pipeline(question=question, context=text_storage[text_id]['text'])
     r = qa_pipeline(question=question, context=text_storage[uuid]['text'], top_k=10)
-    #print(r)
-    #print('-----------------------------')
     return r[0]
 
 

requirements.txt

@@ -6,5 +6,6 @@ intel_extension_for_pytorch==2.0.100
 torch==2.0.1
 transformers==4.29.2
 pydantic==2.4.2
-pypdf2==3.0.1
-python-multipart==0.*
+pypdf2==3.0.*
+python-multipart==0.*
+newspaper3k==0.*

static/index.html

@@ -26,6 +26,8 @@
     <link rel="icon" type="image/png" sizes="16x16" href="fav/favicon-16x16.png">
     <link rel="manifest" href="fav/site.webmanifest">
     <link rel="mask-icon" href="fav/safari-pinned-tab.svg" color="#5bbad5">
+    <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/toastr.js/latest/toastr.min.js"></script>
   </head>
   <body>
 
@@ -99,7 +101,13 @@
                 <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="{{button_close_modal}"></button>
             </div>
             <div class="modal-body">
-                <textarea id="textArea" rows="4" cols="50"></textarea>
+                <div class="alert alert-info" role="alert">
+                  Please enter a text of maximum 4000 characters.
+                </div>
+                <textarea id="textArea" class="form-control" rows="4" maxlength="4000"></textarea>
+                <div class="mt-2">
+                  <span id="charCount">4000</span> characters remaining
+                </div>
             </div>
             <div class="modal-footer">
               <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{{button_close_modal}}</button>
@@ -117,8 +125,17 @@
               <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="{{button_close_modal}"></button>
             </div>
             <div class="modal-body">
+              <div class="alert alert-info" role="alert">
+                You can upload a PDF or TXT file, maximum 1MB in size.<br>
+                Only the first 4000 characters of the text will be processed.
+              </div>
               <form id="file-form" enctype="multipart/form-data">
-                <input id="fileInput" type="file" name="file" accept=".pdf">
+                <div class="input-group mb-3">
+                    <div class="custom-file">
+                        <input type="file" class="custom-file-input" id="fileInput" name="file" accept=".pdf,.txt">
+                        <label class="custom-file-label" for="fileInput">Choose file</label>
+                    </div>
+                </div>
               </form>
             </div>
             <div class="modal-footer">
@@ -129,6 +146,30 @@
     </div>
   </div>
 
+  <div class="modal fade" tabindex="-1" id="modalUrl">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title">Enter the Webpage URL</h5>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="{{button_close_modal}"></button>
+            </div>
+            <div class="modal-body">
+              <div class="alert alert-info" role="alert">
+                You can input a URL from a news website.<br>Note that with other types of websites, the functionality may not work as effectively.
+              </div>
+              <div class="form-group">
+                <label for="url">URL:</label>
+                <input type="text" class="form-control" id="url" value="">
+              </div>
+            </div>
+            <div class="modal-footer">
+              <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">{{button_close_modal}}</button>
+              <button type="button" class="btn btn-primary" id="sendButton3">Send</button>
+            </div>
+        </div>
+    </div>
+  </div>
+
   <header class="hide-section">
     <div class="container">
       <div class="row">

static/js/app.js

@@ -595,13 +595,27 @@ function hideChat() {
 }
 
 // Agrega el evento para el botón "Enviar"
-document.getElementById('sendButton').addEventListener('click', function () {
+$('#sendButton').click(function (evt) {
+	evt.preventDefault();
+
 	var textData = {
 		text: $('#textArea').val(),
-		pdf: null,
-		html_url: null
 	};
 
+	// Configurar la posición de Toastr en la parte superior
+	toastr.options.positionClass = 'toast-top-center';
+
+	// Verificar si la variable de texto está vacía
+    if (textData.text.trim() === '') {
+        toastr.error("Error: Text cannot be empty.");
+        return;
+    }
+
+	// Deshabilitar el botón y agregar un spinner
+    var sendButton = $('#sendButton');
+    sendButton.prop('disabled', true);
+    sendButton.html('<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Sending...');
+
 	// Realiza una llamada POST al endpoint /store_text
 	$.ajax({
 		type: "POST",
@@ -609,41 +623,147 @@ document.getElementById('sendButton').addEventListener('click', function () {
 		data: JSON.stringify(textData),
 		contentType: "application/json",
 		success: function (data) {
+			// Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+
 			$('#textArea').val('');
 			// Cierra el modal después de enviar el texto
 			textModal.hide();
 			displayChat(chatId);
-		}
+		},
+        error: function (xhr, status, error) {
+            // Verificar si hay un código de error del backend
+            if (xhr.status === 400 || xhr.status === 500) {
+                toastr.error(`Error: ${xhr.status} - ${error}`);
+            } else {
+                toastr.error("Error: Connection refused. Please try again later.");
+            }
+
+            // Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+        }
 	});
 });
 
 $('#sendButton2').click(function (evt) {
-	evt.preventDefault();
-	var formData = new FormData($('#file-form')[0]);
+    evt.preventDefault();
+    var formData = new FormData($('#file-form')[0]);
+    var sendButton = $('#sendButton2');
+
+    // Configurar la posición de Toastr en la parte superior
+    toastr.options.positionClass = 'toast-top-center';
+
+    var fileInput = $('#fileInput')[0];
+    var fileSize = fileInput.files[0].size; // Tamaño en bytes
+    var maxSize = 1*1024*1024; // 1MB en bytes
+
+    // Validar el tamaño del archivo
+    if (fileSize > maxSize) {
+        toastr.error('Error: File size exceeds 1MB limit.');
+        return;
+    }
+
+    // Deshabilitar el botón y agregar un spinner
+    sendButton.prop('disabled', true);
+    sendButton.html('<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Uploading...');
+
+    $.ajax({
+        url: `/upload_file/${uuid}`,
+        type: 'POST',
+        data: formData,
+        async: false,
+        cache: false,
+        contentType: false,
+        processData: false,
+        success: function (data) {
+            $('#fileInput').val('');
+            // Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+
+            // Cierra el modal después de enviar el texto
+            textModal.hide();
+            displayChat(chatId);
+        },
+        error: function (xhr, status, error) {
+            // Mostrar mensaje de error con Toastr
+            toastr.error('Error uploading the file');
+
+            // Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+        }
+    });
+});
 
-	$.ajax({
-		url: `/upload_file/${uuid}`,
-		type: 'POST',
-		data: formData,
-		async: false,
-		cache: false,
-		contentType: false,
-		processData: false,
-		success: function (data) {
-			$('#fileInput').val('');
-			// Cierra el modal después de enviar el texto
-			textModal.hide();
-			displayChat(chatId);
-		},
-		error: function (error) {
-			console.log(error);
-			alert('Error al cargar el archivo');
-		}
-	});
+
+$('#sendButton3').click(function () {
+    var textData = {
+        html_url: $('#url').val(),
+    };
+
+	// Configurar la posición de Toastr en la parte superior
+	toastr.options.positionClass = 'toast-top-center';
+
+    var sendButton = $('#sendButton3');
+
+    // Verificar si la variable de texto está vacía
+    if (textData.html_url.trim() === '') {
+        toastr.error("Error: URL cannot be empty.");
+        return;
+    }
+
+	// Validar la URL
+    var urlRegex = new RegExp('^(https?:\\/\\/)?'+ 
+    '((([a-z\\d]([a-z\\d-]*[a-z\\d])*)\\.)+[a-z]{2,}|'+ 
+    '((\\d{1,3}\\.){3}\\d{1,3}))'+ 
+    '(\\:\\d+)?(\\/[-a-z\\d%_.~+]*)*'+ 
+    '(\\?[;&a-z\\d%_.~+=-]*)?'+ 
+    '(\\#[-a-z\\d_]*)?$','i');
+    if (!urlRegex.test(textData.html_url)) {
+        toastr.error("Error: Invalid URL.");
+        return;
+    }
+
+    // Deshabilitar el botón y agregar un spinner
+    sendButton.prop('disabled', true);
+    sendButton.html('<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Sending...');
+
+    // Realiza una llamada POST al endpoint /store_text
+    $.ajax({
+        type: "POST",
+        url: `/store_text/${uuid}`,
+        data: JSON.stringify(textData),
+        contentType: "application/json",
+        success: function (data) {
+            $('#url').val('');
+			// Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+			
+            // Cierra el modal después de enviar el texto
+            textModal.hide();
+            displayChat(chatId);
+        },
+        error: function (xhr, status, error) {
+            if (xhr.status === 0) {
+                toastr.error('Error: Connection refused. Please try again later.');
+            } else if (xhr.status === 400 || xhr.status === 500) {
+                toastr.error(`Error: ${xhr.status} - ${error}`);
+            }
+
+            // Habilitar el botón nuevamente
+            sendButton.prop('disabled', false);
+            sendButton.html('Send');
+        }
+    });
 });
 
 
 
+
 $(document).delegate(".start-chat", "click", function () {
 	chatId = $(this).attr("data-index");
 	if (chatId == 0) {
@@ -656,6 +776,11 @@ $(document).delegate(".start-chat", "click", function () {
 			keyboard: false
 		});
 		textModal.show();
+	} else if (chatId == 2) {
+		textModal = new bootstrap.Modal(document.getElementById('modalUrl'), {
+			keyboard: false
+		});
+		textModal.show();
 	}
 	//console.log(chatId);
 	//displayChat($(this).attr("data-index"));
@@ -1474,4 +1599,12 @@ $(document).ready(function () {
 		localStorage.setItem('text-talk-settings', JSON.stringify(settings));
 		$('#modalConfig').modal('hide');
 	});
+
+	// Maneja el conteo de caracteres
+	$('#textArea').on('input', function () {
+		var maxLength = 4000;
+		var currentLength = $(this).val().length;
+		var remaining = maxLength - currentLength;
+		$('#charCount').text(remaining);
+	});
 });

static/json/prompts-en.json

@@ -17,7 +17,7 @@
     "widget_name": "{{widget_name_2}}",
     "image": "img/icon-file.svg",
     "welcome_message": "Welcome! Feel free to ask any questions about the selected text. I'm here to provide you with insightful answers.",
-    "description": "<h6>Tips:</h6> Haz las preguntas que quieras acerca del texto",
+    "description": "Our mission is to simplify text processing and enhance information retrieval using cutting-edge LLM models. Explore our project on <a href=\"https://github.com/roaltopo/scan-u-doc\">GitHub</a>.",
     "display_welcome_message": true,
     "chat_minlength": 2,
     "chat_maxlength": 1000,
@@ -29,7 +29,7 @@
     "widget_name": "{{widget_name_3}}",
     "image": "img/icon-webpage.svg",
     "welcome_message": "Welcome! Feel free to ask any questions about the selected text. I'm here to provide you with insightful answers.",
-    "description": "<h6>Tips:</h6> Haz las preguntas que quieras acerca del texto",
+    "description": "Our mission is to simplify text processing and enhance information retrieval using cutting-edge LLM models. Explore our project on <a href=\"https://github.com/roaltopo/scan-u-doc\">GitHub</a>.",
     "display_welcome_message": true,
     "chat_minlength": 3,
     "chat_maxlength": 1000,