Update main.py

main.py

@@ -8,6 +8,7 @@ from crawl4ai import AsyncWebCrawler
 from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, LLMExtractionStrategy
 import json
 import logging
+import trafilatura
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -77,6 +78,47 @@ async def test_url(api_key: str = Depends(verify_api_key), url: str = Query(...,
     result = await simple_crawl(url=url)
     return {"markdown": result.markdown}
 
+
+@app.get("/basic-crawl-article")
+def extract_article(
+    url: str,
+    record_id: Optional[str] = Query(None, description="Add an ID to the metadata."),
+    no_fallback: Optional[bool] = Query(False, description="Skip the backup extraction with readability-lxml and justext."),
+    favor_precision: Optional[bool] = Query(False, description="Prefer less text but correct extraction."),
+    favor_recall: Optional[bool] = Query(False, description="When unsure, prefer more text."),
+    include_comments: Optional[bool] = Query(True, description="Extract comments along with the main text."),
+    output_format: Optional[str] = Query('txt', description="Define an output format: 'csv', 'json', 'markdown', 'txt', 'xml', 'xmltei'.", enum=["csv", "json", "markdown", "txt", "xml", "xmltei"]),
+    target_language: Optional[str] = Query(None, description="Define a language to discard invalid documents (ISO 639-1 format)."),
+    include_tables: Optional[bool] = Query(True, description="Take into account information within the HTML <table> element."),
+    include_images: Optional[bool] = Query(False, description="Take images into account (experimental)."),
+    include_links: Optional[bool] = Query(False, description="Keep links along with their targets (experimental)."),
+    deduplicate: Optional[bool] = Query(False, description="Remove duplicate segments and documents."),
+    max_tree_size: Optional[int] = Query(None, description="Discard documents with too many elements.")
+):
+    response = await simple_crawl(url=url)
+    filecontent = response.html
+    extracted = trafilatura.extract(
+        filecontent,
+        url=url,
+        record_id=record_id,
+        no_fallback=no_fallback,
+        favor_precision=favor_precision,
+        favor_recall=favor_recall,
+        include_comments=include_comments,
+        output_format=output_format,
+        target_language=target_language,
+        include_tables=include_tables,
+        include_images=include_images,
+        include_links=include_links,
+        deduplicate=deduplicate,
+        max_tree_size=max_tree_size
+    )
+    
+    if extracted:
+        return {"article": trafilatura.utils.sanitize(extracted)}
+    else:
+        return {"error": "Could not extract the article"}
+
 @app.get("/test")
 async def test(api_key: str = Depends(verify_api_key)):
     result = await simple_crawl("https://www.nbcnews.com/business")