Update app.py

app.py

@@ -1,32 +1,49 @@
 import gradio as gr
 import os
 import openai
-import newspaper
+from newspaper import Article
 import json
 import re
 from transformers import GPT2Tokenizer
+import nltk
+from nltk.tokenize import sent_tokenize
+import requests
+
+nltk.download('punkt')
 
 
 # define the text summarizer function
 def text_prompt(request, page_url, contraseña, temp):
     try:
-        page = newspaper.Article(url=page_url)
-        page.download()
+        headers = {'User-Agent': 'Chrome/83.0.4103.106'}
+        response = requests.get(page_url, headers=headers)
+        html = response.text
+
+        page = Article('')
+        page.set_html(html)
         page.parse()
+
     except Exception as e:
         return "", f"--- Ha ocurrido un error al procesar la URL: {e} ---", ""
-
-    tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-
-    tokens = tokenizer.tokenize(page.text)
     
-    # Recortar el texto a un máximo de 1000 tokens
-    num_tokens = len(tokens)
+    tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+    sentences = sent_tokenize(page.text)
     
-    if num_tokens > 1800:
-        tokens = tokens[:1800]
+    tokens = []
+    page_text = ""
     
-    page_text = " ".join(tokens)
+    for sentence in sentences:
+        tokens.extend(tokenizer.tokenize(sentence))
+        
+        # Recortar el texto a un máximo de 1800 tokens
+        if len(tokens) > 1800:
+            break
+        page_text += sentence + " "
+        
+    # Eliminar el ultimo espacio
+    page_text = page_text.strip()
+
+    num_tokens = len(tokens)
 
     if num_tokens > 10:
         openai.api_key = contraseña
@@ -41,9 +58,11 @@ def text_prompt(request, page_url, contraseña, temp):
             )
             # get the response text
             response_text = response.choices[0].text
+            total_tokens = response["usage"]["total_tokens"]
+
             # clean the response text
             response_text = re.sub(r'\s+', ' ', response_text)
-            return page.text, response_text, num_tokens
+            return page.text, response_text, total_tokens
         except Exception as e:
             return page.text, f"--- Ha ocurrido un error al procesar la solicitud: {e} ---", num_tokens
     return page.text, "--- Min number of tokens ---", num_tokens
@@ -56,7 +75,7 @@ iface = gr.Interface(
             gr.Textbox(lines=1, placeholder="Enter your API-key here...", label="API-Key:", type="password"),
             gr.Slider(0.0,1.0, value=0.3, label="Temperature:")
             ],
-    outputs=[gr.Textbox(label="Input:"), gr.Textbox(label="Output:"), gr.Textbox(label="Tokens:")],
+    outputs=[gr.Textbox(label="Input:"), gr.Textbox(label="Output:"), gr.Textbox(label="Total Tokens:")],
     examples=[["Summarize the following text as a list:","https://blog.google/outreach-initiatives/google-org/our-commitment-on-using-ai-to-accelerate-progress-on-global-development-goals/","",0.3],
             ["Generate a summary of the following text. Give me an overview of main business impact from the text following this template:\n- Summary:\n- Business Impact:\n- Companies:", "https://ai.googleblog.com/2019/10/quantum-supremacy-using-programmable.html","",0.7],
             ["Generate the next insights based on the following text. Indicates N/A if the information is not available in the text.\n- Summary:\n- Acquisition Price:\n- Why is this important for the acquirer:\n- Business Line for the acquirer:\n- Tech Focus for the acquired (list):","https://techcrunch.com/2022/09/28/eqt-acquires-billtrust-a-company-automating-the-invoice-to-cash-process-for-1-7b/","",0.3]