Remove NLTK package

app/main.py

@@ -39,6 +39,7 @@ logging.basicConfig(
 
 logger = logging.getLogger(__name__)
 
+
 # Initialize handlers at startup
 message_handler = None
 webhook_handler = None

app/search/bm25_search.py

@@ -32,9 +32,9 @@ class BM25_search:
         - perform_lemmatization (bool): Whether to perform lemmatization on tokens.
         """
         # Ensure NLTK resources are downloaded only once
-        if not BM25_search.nltk_resources_downloaded:
-            download_nltk_resources()
-            BM25_search.nltk_resources_downloaded = True  # Mark as downloaded
+        # if not BM25_search.nltk_resources_downloaded:
+        #     download_nltk_resources()
+        #     BM25_search.nltk_resources_downloaded = True  # Mark as downloaded
 
         self.documents: List[str] = []
         self.doc_ids: List[str] = []
@@ -153,7 +153,7 @@ async def initialize_bm25_search(remove_stopwords: bool = True, perform_lemmatiz
     Initializes the BM25search with proper NLTK resource downloading.
     """
     loop = asyncio.get_running_loop()
-    await loop.run_in_executor(None, download_nltk_resources)
+    # await loop.run_in_executor(None, download_nltk_resources)
     return BM25_search(remove_stopwords, perform_lemmatization)
 
 

indexed_links.txt

@@ -1,55 +0,0 @@
-2025-01-02 09:49:36,018 - INFO - 7d4efdfd-82bd-4a05-98b3-c70de9c16e0b_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 09:49:36,486 - INFO - d804a237-9b3e-4c83-8871-2aae6ca1d73d_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 09:49:37,089 - INFO - 820b8df1-2492-4ac6-9230-650bbb875122_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 09:49:37,666 - INFO - 82da1b85-5bc3-42ed-80ee-e9b7d8e44123_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 09:49:38,270 - INFO - d18fa576-c9cc-4be3-a2b4-7ab46a392f2b_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-02 09:58:45,530 - INFO - b05153a2-968c-4e08-9113-07b51df51d2e_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 09:58:46,083 - INFO - 35884a24-4095-4ecd-b1c3-10c88947e12b_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 09:58:46,693 - INFO - 8087024d-1e08-4cae-a336-dfe591ae27ce_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 09:58:47,195 - INFO - 9bdd88b3-4ed8-4f89-9abb-e2bc49b4da0f_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 09:58:47,592 - INFO - ff0a3e9f-cd5a-4977-9ab6-721179f41a9b_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-02 10:02:56,391 - INFO - 7fad3aab-6cbf-40ba-91d6-1213a113e4cc_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 10:02:56,993 - INFO - 38ac11a0-fd15-438d-985f-3bbe670f60cf_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 10:02:57,542 - INFO - 58a96911-2ee0-438b-adc6-62de8c6ef8c0_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 10:02:58,086 - INFO - 87436ad6-0d9a-4374-91b6-1a95c8a623e5_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 10:02:58,853 - INFO - cf72c268-2bed-4ab3-b1de-3371c9a540fa_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-02 14:05:00,442 - INFO - 5bf4bf35-ebc1-418a-a14b-f2c536983b33_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 14:05:01,488 - INFO - afaa8e05-d2c4-4b4f-96b2-894bd873ac7c_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 14:05:02,011 - INFO - 10a2becc-d166-4a4a-b81f-f8f52e8da3a2_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 14:05:02,462 - INFO - 7da6b309-accf-4370-9877-3852fa0eb1ad_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 14:05:03,041 - INFO - 5c3904bf-f805-4710-9184-24d5871c0687_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-02 15:10:46,920 - INFO - ad2b962e-ff55-4a90-b33e-f8a79628a0e6_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 15:10:48,607 - INFO - 4d9d2b38-c5bc-4525-bbb1-3f9fb7974156_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 15:10:49,220 - INFO - 4884a10b-505b-4c8b-9fd8-53e9f8d959d2_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 15:10:49,841 - INFO - 4e62d8fc-cde4-4e59-874c-bd5b86b208bb_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 15:10:50,745 - INFO - e6e37e7f-525d-435c-b860-58f719995032_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-02 15:12:11,852 - INFO - 7b8f7a8a-b10a-4d2b-b8af-33d871c6883f_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-02 15:12:12,247 - INFO - 8d6d7eed-cf50-4dd0-9196-b2b04df9aa25_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-02 15:12:12,789 - INFO - 2f0997a8-26e4-4459-95d1-f022b75058cc_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-02 15:12:13,822 - INFO - 73b52f33-9dbf-47a6-8fd6-7c70b0839265_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-02 15:12:14,609 - INFO - 6bf5a433-637f-480a-b801-cc7a66f92e71_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-07 14:14:02,625 - INFO - 363bdd80-43d5-40fe-84b6-a2e89d58e1cf_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-07 14:14:04,956 - INFO - 319b2118-5c37-47e9-a15a-56a1ba28f533_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-07 14:14:05,658 - INFO - 5ce111cb-2cca-4f16-8712-19adf801193d_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-07 14:14:06,171 - INFO - d2688d0d-c09a-427d-97b2-5a6073ef9c06_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-07 14:14:06,767 - INFO - dfa4ed93-a644-4431-81a3-3916924df457_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-01-07 14:18:34,502 - INFO - 9468b228-0fcc-4942-9a0c-559253683f5d_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-01-07 14:18:36,034 - INFO - b390676d-9e05-4e4e-8018-815265d44cba_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-01-07 14:18:37,236 - INFO - f20259fd-f923-4c0c-9cf0-956a8f7c438e_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-01-07 14:18:40,388 - INFO - ad6cbcf3-9662-48e0-88d8-6e4eca796842_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-01-07 14:18:41,380 - INFO - b4df84cf-55f5-46d2-b9cb-f8039fb102e7_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-02-12 12:58:48,544 - INFO - a3d8ff96-c21b-46ae-a6f5-d3d7b9952e01_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-02-12 12:58:49,147 - INFO - 9100ac6e-2da9-4b23-8560-d0e0f2129100_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-02-12 12:58:49,574 - INFO - d8c6373e-2de0-4c22-bb4f-394a66dcf397_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-02-12 12:58:50,396 - INFO - 67e0bd19-4dca-4666-ac82-2c161482df43_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-02-12 12:58:50,966 - INFO - 37c33e40-db31-430d-9e80-5062fd0c5b6a_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-02-12 13:03:18,154 - INFO - f6aa8d0a-ea7d-46ef-92f5-39506ecd439c_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-02-12 13:03:18,609 - INFO - 7ebd203f-717c-43e0-90b3-d8de87207c5c_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-02-12 13:03:19,892 - INFO - 8410df14-2343-49c5-b810-e9c974d3a1b4_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-02-12 13:03:24,692 - INFO - dc571c48-21ec-4cc7-a845-cc89254f82bb_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-02-12 13:03:27,569 - INFO - dd340448-b640-4ffa-8735-8b984bde825f_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda
-2025-02-12 13:07:30,320 - INFO - e162050f-e78d-424b-94c1-67225d392763_https://sswalfa.surabaya.go.id/info/detail/izin-pengumpulan-sumbangan-bencana
-2025-02-12 13:07:30,753 - INFO - 7d64ed75-53a7-41c7-9b64-5767f95e780a_https://sswalfa.surabaya.go.id/info/detail/izin-pemakaian-ruang-terbuka-hijau
-2025-02-12 13:07:44,452 - INFO - a9ccda4f-470a-4d0b-b6b1-2a993a30b62a_https://sswalfa.surabaya.go.id/info/detail/pengganti-ipt
-2025-02-12 13:07:49,413 - INFO - 6cec7f22-f829-4ebc-a4e3-fcf07d12e264_https://sswalfa.surabaya.go.id/info/detail/arahan-sistem-drainase
-2025-02-12 13:07:49,730 - INFO - ab810981-4182-4b53-9b2e-de0423980e64_https://sswalfa.surabaya.go.id/info/detail/rangkaian-pelayanan-surat-pernyataan-belum-menikah-lagi-bagi-jandaduda