new

backend/__init__.py

@@ -0,0 +1 @@
+import backend.tests

backend/invoke_worker/__init__.py

@@ -2,10 +2,8 @@ from core.settings import DEBUG
 import sys
 
 if len(sys.argv) > 1 and not (sys.argv[1] in ['migrate', "makemigrations", "clear_all_sessions"]):
-    print("[Worker] Starting Thread...")
     from backend.invoke_worker import (
         session,
         chapter_queue,
     )
-    print("[Worker] Thread Started!")
 

backend/invoke_worker/chapter_queue.py

@@ -13,6 +13,7 @@ from asgiref.sync import async_to_sync
 
 from django.db.models import Count
 
+
 import requests, environ, os, subprocess, shutil, zipfile, uuid, sys
 
 env = environ.Env()
@@ -100,19 +101,16 @@ class Job(Thread):
                             job = web_scrap.source_control[source].get_chapter.scrap(comic_id=comic_id,chapter_id=chapter_id,output_dir=input_dir)
                             if job.get("status") == "success":
                                 
-                                # with open(os.path.join(LOG_DIR,"image_translator_output.log"), "w") as file:
-                                result = subprocess.run(
-                                    script,
-                                    cwd=os.path.join(BASE_DIR, "backend", "module", "utils", "image_translator"),
-                                    check=True,
-                                    shell=False if sys.platform.startswith('linux') else True,
-                                    # stdout=file,
-                                    # stderr=file,
-                                    text=True,
-                                )
-                                
-                                
-                                
+                                with open(os.path.join(LOG_DIR,"image_translator_output.log"), "w") as file:
+                                    result = subprocess.run(
+                                        script,
+                                        cwd=os.path.join(BASE_DIR, "backend", "module", "utils", "image_translator"),
+                                        check=True,
+                                        shell=False if sys.platform.startswith('linux') else True,
+                                        stdout=file,
+                                        stderr=file,
+                                        text=True,
+                                    )
                                 if result.returncode != 0: raise Exception("Image Translator Execution error!")
                                 os.makedirs(managed_output_dir,exist_ok=True)
                                 shutil.rmtree(input_dir)

backend/module/web_scrap/ColaManga/get.py

@@ -33,7 +33,7 @@ def scrap(id:int=1):
         cover_url = driver.find_element(By.CLASS_NAME, "fed-list-pics").get_attribute("data-original")
         cover_url_split = cover_url.split("/")
         cover_id = cover_url_split[len(cover_url_split)-2]
-        DATA["cover"] = f"/api/web_scrap/colamanga/get_cover/{id}/{cover_id}/"
+        DATA["cover"] = f"/api/web_scrap/get_cover/colamanga/{id}/{cover_id}/"
         
         
         content_info_element = driver.find_element(By.CLASS_NAME, "fed-deta-content")

backend/module/web_scrap/ColaManga/get_chapter.py

@@ -17,36 +17,24 @@ scraper = None
 
 def __scrollToBottom(driver:object=None):
     if not driver: raise ValueError("The 'driver' argument is required.")
-    timeout = 10
-    interval = [0]
-
-    def timer(interval,timeout):
-        while True:
-            time.sleep(1)
-            interval[0] = interval[0]+1
-            if interval[0] >= timeout: break
-    t = threading.Thread(target=timer, args=[interval,timeout])
-    t.daemon = True
-    t.start()
+    
+    timeout = date_utils.utc_time().add(60,'second').get()
     
     previous_height = 0
     scrolledY = 0
     while True:
+        if date_utils.utc_time().get() >= timeout: raise Exception("[Get Chapter] Finding lastest element Timed out!")
         # Scroll to the bottom of the page
         driver.execute_script(f"window.scrollBy(0, {scrolledY});")
-        time.sleep(0.25)
         
         current_height = driver.execute_script("return document.documentElement.scrollHeight")
         
         if current_height > previous_height: 
             previous_height = current_height
-            interval[0] = 0
         else:
             parent_div = driver.find_element(By.CLASS_NAME, "mh_mangalist")
             child_elements = parent_div.find_elements(By.XPATH, "./*")
-            if child_elements[-1].tag_name != 'script' and child_elements[-1].get_attribute('text') != '__cad.read_periodical();':
-                if interval[0] >= timeout: break
-            else: interval[0] = 0
+            if child_elements[-1].get_attribute('text') != '__cad.read_periodical();': break
         scrolledY += 50
         
 
@@ -71,24 +59,28 @@ def scrap(comic_id:str="",chapter_id:str="",output_dir:str=""):
         child_list = parent_element.find_elements(By.CLASS_NAME, "mh_comicpic")
         
         blob_list = []
-        
+        is_no_more = False
         for child in child_list:
-            image_element = child.find_element(By.TAG_NAME, "img")
-            url = image_element.get_attribute("src")
-            if not url: continue
-            if url.split(":")[0] == "blob":
-                timeout = 0
-                while True:
-                    if timeout >= MAX_TIMEOUT: raise Exception('#1 Timed out!')
+            if is_no_more: break
+            timeout = date_utils.utc_time().add(10,'second').get()
+            while True:
+                if date_utils.utc_time().get() >= timeout: 
+                    is_no_more = True
+                    break
+                image_element = child.find_element(By.TAG_NAME, "img")
+                driver.execute_script("arguments[0].scrollIntoView({ behavior: 'smooth', block: 'center' });", image_element)
+                
+                url = image_element.get_attribute("src")
+                if url: 
                     is_image_loaded = driver.execute_script(
                         "return arguments[0].complete", 
                         image_element
                     )
-                    if is_image_loaded: break
+                    if is_image_loaded: 
+                        blob_list.append(url)
+                        break
+                    
                     
-                    timeout += 1
-                    time.sleep(1)
-                blob_list.append(url)
         
         
         def process_browser_log_entry(entry):

backend/module/web_scrap/ColaManga/get_list.py

@@ -40,7 +40,7 @@ def scrap(orderBy:str="monthlyCount",page:int=1):
             object["id"] = id
             cover_link_split = li.find("a", {"class": "fed-list-pics"}).get("data-original").split("/")
             cover_id = cover_link_split[len(cover_link_split)-2]
-            object["cover"] = f"/api/web_scrap/colamanga/get_cover/{id}/{cover_id}/"
+            object["cover"] = f"/api/web_scrap/get_cover/colamanga/{id}/{cover_id}/"
             
             DATA.append(object)
             

backend/module/web_scrap/ColaManga/search.py

@@ -40,7 +40,7 @@ def scrap(search:str="",page:int=1):
             
             cover_link_split = a.get("data-original").split("/")
             cover_id = cover_link_split[len(cover_link_split)-2]
-            object["cover"] = f"/api/web_scrap/colamanga/get_cover/{id}/{cover_id}/"
+            object["cover"] = f"/api/web_scrap/get_cover/colamanga/{id}/{cover_id}/"
             
             dd = dl.find("dd",{"class": "fed-deta-content"})
 

backend/module/web_scrap/utils.py

@@ -13,7 +13,7 @@ import time, threading
 class SeleniumScraper:
     def __init__(self):
         options = Options()
-        options.add_argument("--headless")
+        # options.add_argument("--headless")
         options.add_argument('--no-sandbox')
         options.add_argument("--no-quit")
         options.add_argument('--disable-extensions')

backend/socket/queue.py

@@ -26,7 +26,7 @@ class RequestChapter(WebsocketConsumer):
         
         SocketRequestChapterQueueCache.objects.filter(socket_id=user_socket_id).update(channel_name = self.channel_name)
         
-        print(f"User: ({user_socket_id}) connected to socket room: ({self.room_id})")
+
         self.accept()
         
         self.send(text_data=json.dumps({
@@ -46,6 +46,5 @@ class RequestChapter(WebsocketConsumer):
     def disconnect(self, close_code):
         user_socket_id = self.scope['url_route']['kwargs']['socket_id']
         
-        print(f"User: ({user_socket_id}) disconnected from socket room: ({self.room_id})")
         
 

backend/tests.py

@@ -1,3 +1,7 @@
 from django.test import TestCase
+# from backend.module.utils import date_utils
+
+
+# current_time = date_utils.utc_time().get()
+# print(current_time > date_utils.utc_time().add(-10,'second').get())
 
-# Create your tests here.

backend/urls.py

@@ -17,7 +17,7 @@ urlpatterns = [
     path('web_scrap/get_list/', web_scrap.get_list),
     path('web_scrap/search/', web_scrap.search),
     path('web_scrap/get/', web_scrap.get),
-    path('web_scrap/<str:source>/get_cover/<str:id>/<str:cover_id>/', web_scrap.get_cover),
+    path('web_scrap/get_cover/<str:source>/<str:id>/<str:cover_id>/', web_scrap.get_cover),
     path('web_scrap/get_chapter/', web_scrap.get_chapter),
     
     

frontend/app/index.tsx

@@ -9,7 +9,7 @@ const Index = () => {
     const pathname = usePathname()
 
     if (pathname === "/" || pathname === "") return (
-        <Redirect href="/view/colamanga/manga-id90484" />
+        <Redirect href="/read/colamanga/manga-nf048578?idx=1" />
     )
 
 }

frontend/app/read/[source]/[comic_id]/index.tsx

@@ -110,7 +110,7 @@ const Index = ({}:any) => {
                     const new_chapter_idx = CHAPTER_IDX.current+1
                     if (imageKeys.slice(-1)[0]?.type !== "no-chapter-banner"){
                         const next_stored_chapter = await ChapterStorage.getByIdx(`${SOURCE}-${COMIC_ID}`,new_chapter_idx, {exclude_fields:["data"]})   
-                        if (next_stored_chapter.data_state === "completed"){
+                        if (next_stored_chapter?.data_state === "completed"){
                             if (temp_image_keys.current.hasOwnProperty(new_chapter_idx)){
                                 delete temp_image_keys.current[new_chapter_idx-2]
                                 const pre_image_keys = imageKeys.filter((data:any) => {

frontend/app/read/components/chapter_image.tsx

@@ -38,6 +38,7 @@ const ChapterImage = ({item, zoom, showOptions,setShowOptions}:any)=>{
                 display:"flex",
                 width:"100%",
                 height:"auto",
+                borderWidth:0,
                 alignItems:"center",
             }}
         >

frontend/app/read/modules/get_chapter.tsx

@@ -25,7 +25,7 @@ export const get_chapter = async (
                     continue; // Skip directories
                 }
                 const fileData = await zipContent.files[fileName].async('base64');
-                file_keys.push({type:"image", idx:CHAPTER_IDX, value: `${CHAPTER_IDX}-${fileName}`})
+                file_keys.push({type:"image", idx:CHAPTER_IDX, id:Number(fileName.split(".")[0]), value: `${CHAPTER_IDX}-${fileName}`})
                 files[`${CHAPTER_IDX}-${fileName}`] = {
                     layout: await getImageLayout("data:image/png;base64," + fileData),
                     data: "data:image/png;base64," + fileData
@@ -46,7 +46,7 @@ export const get_chapter = async (
                     continue; // Skip directories
                 }
                 const fileData = await zipContent.files[fileName].async('base64');
-                file_keys.push({type:"image", idx:CHAPTER_IDX, value: `${CHAPTER_IDX}-${fileName}`})
+                file_keys.push({type:"image", idx:CHAPTER_IDX, id:Number(fileName.split(".")[0]), value: `${CHAPTER_IDX}-${fileName}`})
                 files[`${CHAPTER_IDX}-${fileName}`] = {
                     layout: await getImageLayout("data:image/png;base64," + fileData),
                     data: "data:image/png;base64," + fileData
@@ -54,7 +54,8 @@ export const get_chapter = async (
             }
         }
 
-        file_keys.sort((a, b) => parseInt(a, 10) - parseInt(b, 10))
+        file_keys.sort((a, b) => Number(a.id - b.id))
+        console.log("HUH?",file_keys)
         return {file_keys: file_keys,files:files}
     }
 }

frontend/components/Image.tsx

@@ -1,4 +1,4 @@
-import { useState, useEffect, useContext, useCallback } from "react"
+import { useState, useEffect, useContext, useCallback, useRef } from "react"
 import { Image as _Image } from 'expo-image';
 import { View } from "react-native"
 import ImageCacheStorage from "@/constants/module/image_cache_storage";
@@ -11,7 +11,8 @@ import { useFocusEffect } from "expo-router";
 
 
 const Image = ({source, style, onError, contentFit, transition, onLoad, onLoadEnd}:any) => {
-    const [imageData, setImageData]:any = useState(null)
+    const [imageLoaded, setImageLoaded]:any = useState(false)
+    const imageData:any = useRef(null)
     const [isError, setIsError]:any = useState(false)
     const {showCloudflareTurnstileContext, setShowCloudflareTurnstileContext}:any = useContext(CONTEXT)
     const controller = new AbortController();
@@ -20,28 +21,37 @@ const Image = ({source, style, onError, contentFit, transition, onLoad, onLoadEn
         (async ()=>{
             if(source.hasOwnProperty("type")){
                 if (source.type === "blob"){
-                    setImageData({uri:await blobToBase64(source.data)})
+                    imageData.current = {uri:await blobToBase64(source.data)}
+                    setImageLoaded(true)
                 }else if (source.type === "base64"){
-                    setImageData({uri:source.data})
+                    imageData.current = {uri:source.data}
+                    setImageLoaded(true)
                 }else if (source.type === "file_path"){
-                    setImageData({uri:source.data})
+                    imageData.current = {uri:source.data}
+                    setImageLoaded(true)
                 }else{
                     setIsError(true)
+                    setImageLoaded(false)
                 }
             } else if (source.hasOwnProperty("uri")){
                 const result:any = await ImageCacheStorage.get(setShowCloudflareTurnstileContext,source.uri,signal);
                 if (result.type === "blob"){
-                    setImageData({uri:await blobToBase64(result.data)})
+                    imageData.current = {uri:await blobToBase64(result.data)}
+                    setImageLoaded(true)
                 }else if (result.type === "base64"){
-                    setImageData({uri:result.data})
+                    imageData.current = {uri:result.data}
+                    setImageLoaded(true)
                 }else if (result.type === "file_path"){
-                    setImageData({uri:result.data})
+                    imageData.current = {uri:result.data}
+                    setImageLoaded(true)
                 }else{
                     setIsError(true)
+                    setImageLoaded(false)
                 }
                 
             }else{
-                setImageData(source)
+                imageData.current = source
+                setImageLoaded(true)
             }
 
             
@@ -73,12 +83,12 @@ const Image = ({source, style, onError, contentFit, transition, onLoad, onLoadEn
                         <Icon source={"refresh-circle"} size={25} color={"yellow"}/>
                     </Button>
                 </View>
-                : <>{imageData
+                : <>{imageLoaded
                     
                 
                     ? <_Image 
                         onError={onError} 
-                        source={imageData} 
+                        source={imageData.current} 
                         style={style}
                         contentFit={contentFit}
                         transition={transition}