Integrated trend topic analysis

app.py

@@ -1,5 +1,4 @@
-import datetime
-import hmac
+from datetime import datetime
 import os
 import uuid
 
@@ -10,6 +9,10 @@ from azure.cosmos import ContainerProxy, CosmosClient
 from bs4 import BeautifulSoup, NavigableString
 from dotenv import load_dotenv
 from st_copy_to_clipboard import st_copy_to_clipboard
+from pytrends.request import TrendReq
+import pytz
+import xml.etree.ElementTree as ET
+import re
 
 load_dotenv()
 
@@ -270,7 +273,7 @@ def get_article_summary(article: str) -> str:
             os.environ.get("SUMMARY_API"),
             headers={
                 "Content-Type": "application/json",
-                "Authorization": os.environ.get("SUMMARY_API_KEY"),
+                "Authorization": "Bearer " + os.environ.get("SUMMARY_API_KEY"),
                 "azureml-model-deployment": "heute-summary-api",
             },
             data={"article": article},
@@ -369,10 +372,14 @@ def on_click_handler_generate_article(**kwargs):
             kwargs["webpage_option"],
         )
         headline = create_headline(created_article, kwargs["webpage_option"])
+   
+        print(headline)
+        print(created_article)
+   
     db_analytics_item = {
         "id": str(uuid.uuid4()),
         "oparation": "article_generation",
-        "timestamp": str(datetime.datetime.now()),
+        "timestamp": str(datetime.now()),
     }
     client: ContainerProxy = st.session_state["db_container"]
     client.create_item(body=db_analytics_item)
@@ -397,7 +404,7 @@ def on_click_handler_generate_generate_article_keywords(**kwargs):
         db_analytics_item = {
             "id": str(uuid.uuid4()),
             "oparation": "article_generation",
-            "timestamp": str(datetime.datetime.now()),
+            "timestamp": str(datetime.now()),
         }
         client: ContainerProxy = st.session_state["db_container"]
         client.create_item(body=db_analytics_item)
@@ -415,7 +422,291 @@ def reset_session_state():
     st.session_state["generated_article"] = ""
     st.session_state["studie_links"] = []
     st.session_state["article_summary"] = ""
+    st.session_state["selection_content_trends_ressort"] = "Alle"
+    st.session_state["trends_realtime_all"] = {}
+    st.session_state["trends_today"] = {}
+    st.session_state["trends_yesterday"] = []
+    st.session_state["content_trend_articles_extracted"] = []
+    st.session_state["content_trend_article_links"] = []
+    st.session_state["webpage_option"] = "Boulevard"
+
+## Trends
+def fetch_trends(**kwargs):
+    timespan = kwargs["timespan"]
+
+    match timespan:
+        case "Echtzeit":
+            fetch_trends_realtime()
+        case "Heute":
+            fetch_trends_today()
+        case "Gestern":
+            fetch_trends_yesterday()
+
+
+def fetch_trends_realtime():
+    pytrend = TrendReq(hl='de-AT', tz=360, timeout=(10,50))
+
+    for ressort_name, ressort_code in RESSORTS.items():
+        trends_realtime = pytrend.realtime_trending_searches(pn='AT', cat=ressort_code, count=5)
+
+        st.session_state["trends_realtime_" + ressort_code] = trends_realtime
+
+
+def fetch_trends_today():
+    pytrend = TrendReq(hl='de-AT', tz=360, timeout=(10,50))
+
+    trends_today = pytrend.today_searches(pn="AT")
+    st.session_state["trends_today"] = trends_today
+
+
+def fetch_trends_yesterday():
+    timezone = 'Europe/Vienna'
+    today = datetime.now(pytz.timezone(timezone)).date()
+    feed = ET.fromstring(requests.get(TRENDS_YESTERDAY_FEED_URL).content)
+    ns = {'ht': 'https://trends.google.de/trends/trendingsearches/daily'}  # Define namespace
+
+    trends = []
+
+    for item in feed.findall(".//item"):
+        pubDate = datetime.strptime(item.find('pubDate').text, '%a, %d %b %Y %H:%M:%S %z').date()
+        # Filter: Überspringe, wenn pubDate heute ist
+        if pubDate == today:
+            continue
+        
+        entry = {
+            'title': item.find('title').text,
+            'pubDate': item.find('pubDate').text,
+            'approx_traffic': item.find('ht:approx_traffic', ns).text if item.find('ht:approx_traffic', ns) is not None else None,
+            'news_items': []
+        }
+
+        for news_item in item.findall('ht:news_item', ns):
+            news_details = {
+                'title': news_item.find('ht:news_item_title', ns).text,
+                'snippet': news_item.find('ht:news_item_snippet', ns).text,
+                'url': news_item.find('ht:news_item_url', ns).text,
+                'source': news_item.find('ht:news_item_source', ns).text
+            }
+            entry['news_items'].append(news_details)
+
+        trends.append(entry)
+
+        st.session_state["trends_yesterday"] = trends
+
+
+def render_trends_realtime(container):
+    ressort = st.session_state["selection_content_trends_ressort"]
+    trends_realtime = st.session_state["trends_realtime_" + RESSORTS[ressort]]
+
+    if trends_realtime == {}:
+        container.info(
+            body="Die Echtzeit-Trends wurden noch nicht geladen. Bitte verwende zunächst die Suche auf der rechten Seite!",
+            icon="ℹ️"
+        )
+    else:
+        container.selectbox(
+            label="Ressort auswählen",
+            options=RESSORTS,
+            placeholder="Bitte auswählen",
+            key="selection_content_trends_ressort",
+        )
+        
+        for trend_count, trend in enumerate(trends_realtime, start=1):
+            with container.expander(f"{trend_count} -- {trend['title']}"):
+                articles = extract_article_details_realtime(trend['articles'])
+                
+                for article_count, article in enumerate(articles, start=1):
+                    key = f"selection_trends_realtime_{ressort}_{trend_count}_{article_count}"
+
+                    st.checkbox(
+                        f"{article_count} -- {article['articleTitle']} [Go To →]({article['url']})",
+                        key=key,
+                        disabled=disable_checkbox(f"selection_trends_realtime_{ressort}", key),
+                        on_change=update_trend_article_list(key, article['url'])
+                    )
+
 
+def render_trends_today(container):
+    trends_today = st.session_state["trends_today"]
+
+    if trends_today == {}:
+        container.info(
+            body="Die heutigen Trends wurden noch nicht geladen. Bitte verwende zunächst die Suche auf der rechten Seite!",
+            icon="ℹ️"
+        )
+
+    for trend_count, trend in enumerate(trends_today, start=1):
+        with container.expander(f"{trend_count} -- {trend['title']['query']} | Generated Traffic: {trend['formattedTraffic']}"):
+            articles = extract_article_details_today(trend['articles'])
+            
+            for article_count, article in enumerate(articles, start=1):
+                key = f"selection_trends_today_{trend_count}_{article_count}"
+
+                st.checkbox(
+                    f"{article_count} -- {article['articleTitle']} [Go To →]({article['url']})",
+                    key=key,
+                    disabled=disable_checkbox("selection_trends_today", key),
+                    on_change=update_trend_article_list(key, article['url'])
+                )
+
+
+def render_trends_yesterday(container):
+    trends_yesterday = st.session_state["trends_yesterday"]
+
+    if trends_yesterday == []:
+        container.info(
+            body="Die gestrigen Trends wurden noch nicht geladen. Bitte verwende zunächst die Suche auf der rechten Seite!",
+           icon="ℹ️"
+        )
+
+    for trend_count, trend in enumerate(trends_yesterday, start=1):
+        with container.expander(f"{trend_count}• {trend['title']} | Generated Traffic: {trend['approx_traffic']}"):
+            st.write(f"Veröffentlichungsdatum : {trend['pubDate']}")
+            
+            for article_count, article in enumerate(trend['news_items'], start=1):
+                key = f"selection_trends_yesterday_{trend_count}_{article_count}"
+
+                st.checkbox(
+                    label=f"{article_count} -- {article['title']} [Go To →]({article['url']})",
+                    key=key,
+                    disabled=disable_checkbox("selection_trends_yesterday", key),
+                    on_change=update_trend_article_list(key, article['url'])
+                )
+
+
+def get_checkbox_states(pattern: str):
+    cb_states = {key: val for key, val in st.session_state.items() if re.search(string=key, pattern=pattern)}
+
+    return cb_states
+    
+
+def disable_checkbox(pattern: str, session_key: bool):
+    if session_key in list(st.session_state.keys()):
+        cb_states = get_checkbox_states(pattern)
+
+        return not cb_states[session_key] and not sum(list(cb_states.values())) < LINKS_MAX_CHECKED 
+    
+    return False
+
+
+def update_trend_article_list(session_key, article_url):
+
+    if session_key in list(st.session_state.keys()):
+        if st.session_state[session_key]:
+            if article_url not in st.session_state["content_trend_article_links"]:
+                st.session_state["content_trend_article_links"].append(article_url)
+        else:
+            if article_url in st.session_state["content_trend_article_links"]:
+                st.session_state["content_trend_article_links"].remove(article_url)
+
+
+## Content extraction
+def extract_text_from_element(element):
+    # Initialisiere einen leeren Textstring
+    text_content = ""
+
+    # Überprüfe, ob das Element ein <p>, <ul> oder <ol>-Tag ist
+    if element.name in ["p", "ul", "ol"]:
+        # Extrahiere den Text des Tags und füge ihn zum Textstring hinzu
+        text_content += element.get_text() + "\n"
+
+        # Überprüfe, ob das Element ein Tag mit Kindern ist (kein Textknoten)
+    if not isinstance(element, NavigableString):
+        # Rekursiv durch jedes Child-Element gehen und den Text hinzufügen
+        for child in element.children:
+            text_content += extract_text_from_element(child)
+
+    return text_content
+        
+        
+def filter_empty_lines(text):
+    # Teile den Text in Zeilen auf
+    lines = text.split("\n")
+
+    # Filtere leere Zeilen heraus
+    non_empty_lines = filter(lambda line: line.strip() != "", lines)
+
+    # Verbinde die nicht leeren Zeilen zu einem String
+    filtered_text = "\n".join(non_empty_lines)
+
+    return filtered_text
+
+
+def extract_article(url):
+    # Webseite herunterladen
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+    }
+    response = requests.get(url, headers=headers)
+
+    # Überprüfen, ob die Anfrage erfolgreich war (Status-Code 200)
+    if response.status_code == 200:
+        # HTML-Inhalt parsen
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # Finden Sie das <article>-Tag (nehmen Sie an, dass es eins gibt)
+        article_tag = soup.find("article")
+
+        if article_tag:
+            # Starte die Rekursion für jedes Child-Element des <article>-Tags
+            extracted_text = extract_text_from_element(article_tag)
+            stripped_text = filter_empty_lines(extracted_text)
+            return stripped_text
+        else:
+            print("Kein <article>-Tag gefunden.")
+            return None
+    else:
+        # Falls die Anfrage nicht erfolgreich war, eine Fehlermeldung ausgeben
+        print(f"Fehler: {response.status_code}")
+        return None
+    
+
+def extract_links(**kwargs):
+    with st.spinner("Extrahiere Informationen aus den Links..."):
+        results = []
+        for link in kwargs["links"]:
+            if link != '':
+                results.append(extract_article(link))
+
+        st.session_state[kwargs["key"]] = results
+
+
+def extract_article_details_realtime(articles):
+    article_details = []
+
+            
+    for article in articles:
+        
+        article_detail = {
+            'url': article['url'],
+            'snippet': article['snippet'],  
+            'articleTitle': article['articleTitle'],
+            'time': article['time'] 
+        }
+
+        article_details.append(article_detail)
+    
+    return article_details
+
+
+def extract_article_details_today(articles):
+    article_details = [] 
+
+    for article in articles:
+        article_detail = {
+            'url': article['url'],
+            'snippet': article['snippet'],  
+            'articleTitle': article['title'], 
+        }
+
+        article_details.append(article_detail)
+
+    return article_details
+
+def get_final_articles():
+    final_trend_articles = [article_content for article_key, article_content in st.session_state.items() if re.search(string=article_key, pattern="content_trend_article_final")]
+
+    return final_trend_articles
 
 if "extracted_articles" not in st.session_state:
     st.session_state["extracted_articles"] = []
@@ -433,8 +724,6 @@ if "selected_page" not in st.session_state:
     st.session_state["selected_page"] = 0
 if "generated_article" not in st.session_state:
     st.session_state["generated_article"] = ""
-if "function_state" not in st.session_state:
-    st.session_state["function_state"] = True
 if "generated_headline" not in st.session_state:
     st.session_state["generated_headline"] = ""
 if "webpage_option" not in st.session_state:
@@ -450,12 +739,26 @@ if "db_container" not in st.session_state:
     db_analytics_item = {
         "id": str(uuid.uuid4()),
         "oparation": "page_load",
-        "timestamp": str(datetime.datetime.now()),
+        "timestamp": str(datetime.now()),
     }
     client.create_item(body=db_analytics_item)
     st.session_state["db_container"] = client
 if "article_summary" not in st.session_state:
     st.session_state["article_summary"] = ""
+if "article_generation_mode" not in st.session_state:
+    st.session_state["article_generation_mode"] = "links"
+if "selection_content_trends_ressort" not in st.session_state:
+    st.session_state["selection_content_trends_ressort"] = "Alle"
+if "trends_realtime_all" not in st.session_state:
+    st.session_state["trends_realtime_all"] = {}
+if "trends_today" not in st.session_state:
+    st.session_state["trends_today"] = {}
+if "trends_yesterday" not in st.session_state:
+    st.session_state["trends_yesterday"] = []
+if "content_trend_articles_extracted" not in st.session_state:
+    st.session_state["content_trend_articles_extracted"] = []
+if "content_trend_article_links" not in st.session_state:
+    st.session_state["content_trend_article_links"] = []
 
 PROCESS_STEPS = [
     "Artikel Extraktion",
@@ -464,6 +767,20 @@ PROCESS_STEPS = [
     "Artikel Ausgabe",
 ]
 
+RESSORTS = {
+    "Alle": "all",
+    "Gesundheit": "m",
+    "Business": "b",
+    "Headlines": "h",
+    "Sport": "s",
+    "Entertainment": "e",
+    "Technik": "t",
+}
+
+TRENDS_YESTERDAY_FEED_URL = 'https://trends.google.de/trends/trendingsearches/daily/rss?geo=AT'
+
+LINKS_MAX_CHECKED = 3
+
 
 # def check_password():
 #     """Returns `True` if the user had the correct password."""
@@ -520,16 +837,23 @@ with st.sidebar:
         "Artikel Generierung mit Links",
         key="article_gen_btn",
         use_container_width=True,
-        on_click=lambda: st.session_state.update({"function_state": True}),
+        on_click=lambda: st.session_state.update({"article_generation_mode": "links"}),
     )
     st.button(
         "Artikel Generierung mit Stichpunkten",
         key="headline_gen_btn",
         use_container_width=True,
-        on_click=lambda: st.session_state.update({"function_state": False}),
+        on_click=lambda: st.session_state.update({"article_generation_mode": "keywords"}),
+    )
+    st.button(
+        label="Artikelgenerierung mit Trendthemenanalyse",
+        key="trends_gen_btn",
+        use_container_width=True,
+        on_click=lambda: st.session_state.update({"article_generation_mode": "trends"})
+
     )
 
-if st.session_state["function_state"]:
+if st.session_state["article_generation_mode"] == "links":
     tab_col1, tab_col2, tab_col3, tab_col4 = st.columns([1, 1, 1, 1])
 
     tab_col1.button(
@@ -719,7 +1043,8 @@ if st.session_state["function_state"]:
         st.button(
             "Neuen Artikel generieren", key="reset_btn", on_click=reset_session_state
         )
-else:
+
+if st.session_state["article_generation_mode"] == "keywords":
     st.write(
         "Bitte trage die Stichpunkte ein, die Du in den Artikel einbauen möchtest. Der Textinput ist essenziell für die Generierung des Artikels."
     )
@@ -780,3 +1105,154 @@ else:
         st.button(
             "Neuen Artikel generieren", key="reset_btn", on_click=reset_session_state
         )
+
+if st.session_state["article_generation_mode"] == "trends":
+    
+    trends_left, trends_right = st.columns([0.8, 0.2])
+
+    trends_right.radio(
+        label="Zeitraum auswählen",
+        options=[
+            "Echtzeit",
+            "Heute",
+            "Gestern"
+        ],
+        key="selection_content_trends_timespan"
+    )
+
+    trends_right.button(
+        label="Suchen",
+        type="primary",
+        on_click=fetch_trends,
+        kwargs={
+            "timespan": st.session_state["selection_content_trends_timespan"]
+        },
+        use_container_width=True
+    )
+
+    trends_timespan = st.session_state["selection_content_trends_timespan"]
+
+    match trends_timespan:
+        case "Echtzeit":
+            render_trends_realtime(trends_left)
+        case "Heute":
+            render_trends_today(trends_left)
+        case "Gestern":
+            render_trends_yesterday(trends_left)
+
+    try:
+        st.button(
+            label="Informationen aus Links extrahieren",
+            on_click=extract_links,
+            use_container_width=True,
+            type="secondary",
+            key="btn_extract_trend_links",
+            kwargs={
+                "key": "content_trend_articles_extracted",
+                "links": st.session_state["content_trend_article_links"]
+            },
+        )
+    except Exception as e:
+        print(f"Fehler beim Extrahieren der Informationen: {str(e)}")
+        st.error(
+            body=f"Sie haben einen oder mehrere Links in einem inkorrekten Format angegeben. Bitte lade diese Seite neu und verwende valide URLs: {str(e)}",
+            icon="🚨",
+        )
+
+    st.write()
+
+    if st.session_state["content_trend_article_links"] != []:
+        st.write("Folgende Informationen konnten aus ihren Artikeln extrahiert werden:")
+
+        for i, link_content in enumerate(st.session_state["content_trend_articles_extracted"]):
+            with st.expander(f"Link {i+1}"):
+                if link_content:
+                    st.text_area(
+                        label="Bitte bearbeiten Sie die Informationen falls notwendig:",
+                        value=link_content,
+                        key="content_trend_article_final_" + str(i + 1)
+                    )
+                else:
+                    st.info(
+                        body="Die Webseite Ihres Artikels blockiert das automatische Extrahieren des Artikels. Wenn Sie den Artikel dennoch verwenden möchten, dann können Sie diesen kopieren und in das untenstehende Textfeld einfügen.",
+                        icon="ℹ️",
+                    )
+                    st.text_area(
+                        "Bitte fügen Sie den Artikel ein:",
+                        value=link_content,
+                        key="content_trend_article_final_" + str(i + 1)
+                    )
+
+        st.write("Artikellänge")
+        st.radio(
+            "Optionen",
+            ["Kurz", "Mittel", "Lang", "SEO", "SEO Plus"],
+            key="length_option",
+        )
+
+        st.text_area(
+            "Füge weitere Informationen für den Prompt hinzu, falls nötig:",
+            key="add_info",
+        )
+
+        st.button(
+            "Artikel generieren",
+            key="article_btn",
+            on_click=on_click_handler_generate_article,
+            kwargs={
+                "length_option": st.session_state["length_option"],
+                "final_articles": get_final_articles(),
+                "add_info": st.session_state["add_info"],
+                "webpage_option": st.session_state["webpage_option"],
+            },
+        )
+
+    
+    if st.session_state["generated_headline"] != "" and st.session_state["generated_article"] != "":
+        st.write(f"**{st.session_state['generated_headline']}**")
+        st.write(st.session_state["generated_article"])
+        
+        st.write("**Zusammenfassung:**")
+        st.write(st.session_state["article_summary"])
+        
+        st.write("Kopieren Sie den Artikel: ")
+        st_copy_to_clipboard(
+            st.session_state["generated_headline"]
+            + "\n"
+            + st.session_state["generated_article"]
+        )
+        
+        if st.session_state["studie_links"]:
+            st.write("Hier sind einige Studien, die relevant sein könnten:")
+            for result in st.session_state["studie_links"]:
+                st.write(f"- [{result['title']}]({result['link']})")
+        else:
+            st.write("Keine relevanten Studien gefunden.")
+
+        if "takeaways" in st.session_state:
+            st.write("Hier sind einige Takeaways die wichtig sein könnten:")
+            st.write(st.session_state["takeaways"])
+
+        if "faq" in st.session_state:
+            st.write("Hier sind FAQs zu dem Artikel:")
+            st.write(st.session_state["faq"])
+
+        st.button(
+            "Relevante Studien finden",
+            on_click=get_related_studies,
+            args=(st.session_state["generated_article"],),
+        )
+
+        st.button(
+            "Key Takeaways generieren",
+            on_click=lambda: get_takeaways(st.session_state["generated_article"]),
+        )
+
+        st.button(
+            "FAQ generieren",
+            on_click=lambda: get_faq(st.session_state["generated_article"]),
+        )
+        
+        st.button(
+            "Neuen Artikel generieren", key="reset_btn", on_click=reset_session_state
+        )