layout improvements and faster outputs

app.py

@@ -1,14 +1,6 @@
-import gradio as gr
-import os 
-import subprocess
-
-from huggingface_hub import snapshot_download
-
-
-REPO_ID='SharkSpace/videos_examples'
-snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
-
 
+import subprocess
+import os  
 if os.getenv('SYSTEM') == 'spaces':
 
     subprocess.call('pip install -U openmim'.split())
@@ -19,8 +11,10 @@ if os.getenv('SYSTEM') == 'spaces':
     subprocess.call('mim install mmdet'.split())
     subprocess.call('pip install opencv-python-headless==4.5.5.64'.split())
     subprocess.call('pip install git+https://github.com/cocodataset/panopticapi.git'.split())
-    
 
+import gradio as gr
+
+from huggingface_hub import snapshot_download
 import cv2 
 import dotenv 
 dotenv.load_dotenv()
@@ -33,177 +27,61 @@ from inference import process_frame
 import os
 import pathlib
 import multiprocessing as mp
-
 from time import time
 
 
-def analize_video_serial(x):
-    print(x)
-    path = '/tmp/test/'
-    os.makedirs(path, exist_ok=True)
-    videos = len(os.listdir(path))
-    path = f'{path}{videos}'
-    os.makedirs(path, exist_ok=True)
-    outname = f'{path}_processed.mp4'
-    if os.path.exists(outname):
-        print('video already processed')
-        return outname
-    cap = cv2.VideoCapture(x)
-    counter = 0
-    import pdb;pdb.set_trace()
-    while(cap.isOpened()):
-        ret, frame = cap.read()
-        yield None, frame 
-        if ret==True:
-            name = os.path.join(path,f'{counter:05d}.png')
-            frame = inference_frame_serial(frame)
-            # write the flipped frame
-            
-            cv2.imwrite(name, frame)
-            counter +=1
-           
-            #yield None,frame 
-        else:
-            break
-    # Release everything if job is finished
-    print(path)
-    os.system(f'''ffmpeg -framerate 20 -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
-    return outname,frame 
-
+REPO_ID='SharkSpace/videos_examples'
+snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
 
+def process_video(input_video, out_fps = 'auto', skip_frames = 7):
+    cap = cv2.VideoCapture(input_video)
 
+    output_path = "output.mp4"
+    if out_fps != 'auto' and type(out_fps) == int:
+        fps = int(out_fps)
+    else:
+        fps = int(cap.get(cv2.CAP_PROP_FPS))
+        if out_fps == 'auto':
+            fps = int(fps / skip_frames)
 
+    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
 
-def analyze_video_parallel(x, skip_frames = 5, 
-                           frame_rate_out = 8, batch_size = 16):
-    print(x)
+    video = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
 
-    #Define path to saved images
-    path = '/tmp/test/'
-    os.makedirs(path, exist_ok=True)
-    
-    # Define name of current video as number of videos in path
-    n_videos_in_path = len(os.listdir(path))
-    path = f'{path}{n_videos_in_path}'
-    os.makedirs(path, exist_ok=True)
+    iterating, frame = cap.read()
+    cnt = 0
     
-    # Define name of output video
-    outname = f'{path}_processed.mp4'
+    while iterating:
+        if (cnt % skip_frames) == 0:
+            # flip frame vertically
+            display_frame = inference_frame_serial(frame)
+            video.write(cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB))
+            print('sending frame')
+            print(cnt)
+            yield cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB), cv2.cvtColor(frame, cv2.COLOR_BGR2RGB), None
+        cnt += 1
+        iterating, frame = cap.read()
     
-    if os.path.exists(outname):
-        print('video already processed')
-        return outname
-    
-    cap = cv2.VideoCapture(x)
-    counter = 0
-    pred_results_all = []
-    frames_all = []
-    while(cap.isOpened()):
-        frames = []
-        #start = time()
-        
-        while len(frames) < batch_size:
-            #start = time()
-            ret, frame = cap.read()
-            if ret == False:
-                break
-            elif counter % skip_frames == 0:
-                frames.append(frame)
-            counter += 1
-
-        #print(f'read time: {time()-start}')
-
-        frames_all.extend(frames)
-
-        # Get timing for inference
-        start = time()
-        print('len frames passed: ', len(frames))
-        
-        if len(frames) > 0:
-            pred_results = inference_frame_par_ready(frames)
-            print(f'inference time: {time()-start}')
-            pred_results_all.extend(pred_results)
+    video.release()
+    yield None, None, output_path
 
-        # break while loop when return of the image reader is False
-        if ret == False:
-            break
-
-    print('exited prediction loop')
-    # Release everything if job is finished
-    cap.release()
-        
-    start = time()
-    pool = mp.Pool(mp.cpu_count()-2)
-    pool_out = pool.map(process_frame, 
-                        list(zip(pred_results_all, 
-                                    frames_all, 
-                                    [i for i in range(len(pred_results_all))])))
-    pool.close()
-    print(f'pool time: {time()-start}')
+with gr.Blocks() as demo:
+    with gr.Row():
+        input_video = gr.Video(label="Input")
+        output_video = gr.Video(label="Output Video")
     
-    start = time()
-    counter = 0
-    for pool_out_tmp in pool_out:
-        name = os.path.join(path,f'{counter:05d}.png')
-        cv2.imwrite(name, pool_out_tmp)
-        counter +=1
-        yield None,pool_out_tmp
-        
-    print(f'write time: {time()-start}')
-
-    # Create video from predicted images
-    print(path)
-    os.system(f'''ffmpeg -framerate {frame_rate_out} -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
-    return outname, pool_out_tmp
-  
+    with gr.Row():
+        processed_frames = gr.Image(label="Live Frame")
+        original_frames = gr.Image(label="Original Frame")
 
-def set_example_image(example: list) -> dict:
-    return gr.Video.update(value=example[0])
-
-def show_video(example: list) -> dict:
-    return gr.Video.update(value=example[0])
-    
-with gr.Blocks(title='Shark Patrol',theme=gr.themes.Soft(),live=True,) as demo:
-    gr.Markdown("Alpha Demo of the Sharkpatrol Oceanlife Detector.")
-    with gr.Tab("Preloaded Examples"):
-        
-        with gr.Row():
-            video_example = gr.Video(source='upload',include_audio=False,stream=True)
-        with gr.Row():
-            paths = sorted(pathlib.Path('videos_example/').rglob('*rgb.mp4'))
-            example_preds = gr.Dataset(components=[video_example],
-                                    samples=[[path.as_posix()]
-                                             for path in paths])
-            example_preds.click(fn=show_video,
-                         inputs=example_preds,
-                         outputs=video_example)
+    with gr.Row():
+        paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
+        samples=[[path.as_posix()] for path in paths if 'raw_videos'  in str(path)]
+        examples = gr.Examples(samples, inputs=input_video)
+        process_video_btn = gr.Button("Process Video")
 
-    with gr.Tab("Test your own Video"):
-        with gr.Row():
-            video_input = gr.Video(source='upload',include_audio=False)
-            #video_input.style(witdh='50%',height='50%')
-            image_temp = gr.Image()
-        with gr.Row():
-            video_output = gr.Video()
-            
-            #video_output.style(witdh='50%',height='50%')
-        
-        video_button = gr.Button("Analyze your Video")
-        with gr.Row():
-            paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
-            example_images = gr.Dataset(components=[video_input],
-                                    samples=[[path.as_posix()]
-                                             for path in paths if 'raw_videos'  in str(path)])
+    process_video_btn.click(process_video, input_video, [processed_frames, original_frames, output_video])
 
-    video_button.click(analize_video_serial, inputs=video_input, outputs=[video_output,image_temp])
-
-    example_images.click(fn=set_example_image,
-                         inputs=example_images,
-                         outputs=video_input)
-
-         
 demo.queue()
-if os.getenv('SYSTEM') == 'spaces':
-    demo.launch(width='40%',auth=(os.environ.get('SHARK_USERNAME'), os.environ.get('SHARK_PASSWORD')))
-else: 
-    demo.launch()
+demo.launch()

app3.py

@@ -1,79 +0,0 @@
-
-import subprocess
-import os  
-if os.getenv('SYSTEM') == 'spaces':
-
-    subprocess.call('pip install -U openmim'.split())
-    subprocess.call('pip install python-dotenv'.split())
-    subprocess.call('pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113'.split())
-    subprocess.call('mim install mmcv>=2.0.0'.split())
-    subprocess.call('mim install mmengine'.split())
-    subprocess.call('mim install mmdet'.split())
-    subprocess.call('pip install opencv-python-headless==4.5.5.64'.split())
-    subprocess.call('pip install git+https://github.com/cocodataset/panopticapi.git'.split())
-
-import gradio as gr
-
-from huggingface_hub import snapshot_download
-import cv2 
-import dotenv 
-dotenv.load_dotenv()
-import numpy as np
-import gradio as gr
-import glob
-from inference import inference_frame,inference_frame_serial
-from inference import inference_frame_par_ready
-from inference import process_frame
-import os
-import pathlib
-import multiprocessing as mp
-from time import time
-
-
-REPO_ID='SharkSpace/videos_examples'
-snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
-
-
-
-    
-
-
-
-def process_video(input_video):
-    cap = cv2.VideoCapture(input_video)
-
-    output_path = "output.mp4"
-
-    fps = int(cap.get(cv2.CAP_PROP_FPS))
-    width  = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
-    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-
-    video = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"mp4v"), fps, (width, height))
-
-    iterating, frame = cap.read()
-    while iterating:
-        # flip frame vertically
-        display_frame = inference_frame_serial(frame)
-        video.write(frame)
-        yield display_frame, None
-        iterating, frame = cap.read()
-
-    video.release()
-    yield display_frame, output_path
-
-with gr.Blocks() as demo:
-    with gr.Row():
-        input_video = gr.Video(label="input")
-        processed_frames = gr.Image(label="last frame")
-        output_video = gr.Video(label="output")
-
-    with gr.Row():
-        paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
-        samples=[[path.as_posix()] for path in paths if 'raw_videos'  in str(path)]
-        examples = gr.Examples(samples, inputs=input_video)
-        process_video_btn = gr.Button("process video")
-
-    process_video_btn.click(process_video, input_video, [processed_frames, output_video])
-
-demo.queue()
-demo.launch()

app_legacy.py

@@ -0,0 +1,209 @@
+import gradio as gr
+import os 
+import subprocess
+
+from huggingface_hub import snapshot_download
+
+
+REPO_ID='SharkSpace/videos_examples'
+snapshot_download(repo_id=REPO_ID, token=os.environ.get('SHARK_MODEL'),repo_type='dataset',local_dir='videos_example')
+
+
+if os.getenv('SYSTEM') == 'spaces':
+
+    subprocess.call('pip install -U openmim'.split())
+    subprocess.call('pip install python-dotenv'.split())
+    subprocess.call('pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113'.split())
+    subprocess.call('mim install mmcv>=2.0.0'.split())
+    subprocess.call('mim install mmengine'.split())
+    subprocess.call('mim install mmdet'.split())
+    subprocess.call('pip install opencv-python-headless==4.5.5.64'.split())
+    subprocess.call('pip install git+https://github.com/cocodataset/panopticapi.git'.split())
+    
+
+import cv2 
+import dotenv 
+dotenv.load_dotenv()
+import numpy as np
+import gradio as gr
+import glob
+from inference import inference_frame,inference_frame_serial
+from inference import inference_frame_par_ready
+from inference import process_frame
+import os
+import pathlib
+import multiprocessing as mp
+
+from time import time
+
+
+def analize_video_serial(x):
+    print(x)
+    path = '/tmp/test/'
+    os.makedirs(path, exist_ok=True)
+    videos = len(os.listdir(path))
+    path = f'{path}{videos}'
+    os.makedirs(path, exist_ok=True)
+    outname = f'{path}_processed.mp4'
+    if os.path.exists(outname):
+        print('video already processed')
+        return outname
+    cap = cv2.VideoCapture(x)
+    counter = 0
+    import pdb;pdb.set_trace()
+    while(cap.isOpened()):
+        ret, frame = cap.read()
+        yield None, frame 
+        if ret==True:
+            name = os.path.join(path,f'{counter:05d}.png')
+            frame = inference_frame_serial(frame)
+            # write the flipped frame
+            
+            cv2.imwrite(name, frame)
+            counter +=1
+           
+            #yield None,frame 
+        else:
+            break
+    # Release everything if job is finished
+    print(path)
+    os.system(f'''ffmpeg -framerate 20 -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
+    return outname,frame 
+
+
+
+
+
+def analyze_video_parallel(x, skip_frames = 5, 
+                           frame_rate_out = 8, batch_size = 16):
+    print(x)
+
+    #Define path to saved images
+    path = '/tmp/test/'
+    os.makedirs(path, exist_ok=True)
+    
+    # Define name of current video as number of videos in path
+    n_videos_in_path = len(os.listdir(path))
+    path = f'{path}{n_videos_in_path}'
+    os.makedirs(path, exist_ok=True)
+    
+    # Define name of output video
+    outname = f'{path}_processed.mp4'
+    
+    if os.path.exists(outname):
+        print('video already processed')
+        return outname
+    
+    cap = cv2.VideoCapture(x)
+    counter = 0
+    pred_results_all = []
+    frames_all = []
+    while(cap.isOpened()):
+        frames = []
+        #start = time()
+        
+        while len(frames) < batch_size:
+            #start = time()
+            ret, frame = cap.read()
+            if ret == False:
+                break
+            elif counter % skip_frames == 0:
+                frames.append(frame)
+            counter += 1
+
+        #print(f'read time: {time()-start}')
+
+        frames_all.extend(frames)
+
+        # Get timing for inference
+        start = time()
+        print('len frames passed: ', len(frames))
+        
+        if len(frames) > 0:
+            pred_results = inference_frame_par_ready(frames)
+            print(f'inference time: {time()-start}')
+            pred_results_all.extend(pred_results)
+
+        # break while loop when return of the image reader is False
+        if ret == False:
+            break
+
+    print('exited prediction loop')
+    # Release everything if job is finished
+    cap.release()
+        
+    start = time()
+    pool = mp.Pool(mp.cpu_count()-2)
+    pool_out = pool.map(process_frame, 
+                        list(zip(pred_results_all, 
+                                    frames_all, 
+                                    [i for i in range(len(pred_results_all))])))
+    pool.close()
+    print(f'pool time: {time()-start}')
+    
+    start = time()
+    counter = 0
+    for pool_out_tmp in pool_out:
+        name = os.path.join(path,f'{counter:05d}.png')
+        cv2.imwrite(name, pool_out_tmp)
+        counter +=1
+        yield None,pool_out_tmp
+        
+    print(f'write time: {time()-start}')
+
+    # Create video from predicted images
+    print(path)
+    os.system(f'''ffmpeg -framerate {frame_rate_out} -pattern_type glob -i '{path}/*.png'  -c:v libx264 -pix_fmt yuv420p {outname} -y''')
+    return outname, pool_out_tmp
+  
+
+def set_example_image(example: list) -> dict:
+    return gr.Video.update(value=example[0])
+
+def show_video(example: list) -> dict:
+    return gr.Video.update(value=example[0])
+    
+with gr.Blocks(title='Shark Patrol',theme=gr.themes.Soft(),live=True,) as demo:
+    gr.Markdown("Alpha Demo of the Sharkpatrol Oceanlife Detector.")
+    with gr.Tab("Preloaded Examples"):
+        
+        with gr.Row():
+            video_example = gr.Video(source='upload',include_audio=False,stream=True)
+        with gr.Row():
+            paths = sorted(pathlib.Path('videos_example/').rglob('*rgb.mp4'))
+            example_preds = gr.Dataset(components=[video_example],
+                                    samples=[[path.as_posix()]
+                                             for path in paths])
+            example_preds.click(fn=show_video,
+                         inputs=example_preds,
+                         outputs=video_example)
+
+    with gr.Tab("Test your own Video"):
+        with gr.Row():
+            video_input = gr.Video(source='upload',include_audio=False)
+            #video_input.style(witdh='50%',height='50%')
+            image_temp = gr.Image()
+        with gr.Row():
+            video_output = gr.Video()
+            
+            #video_output.style(witdh='50%',height='50%')
+        
+        video_button = gr.Button("Analyze your Video")
+        with gr.Row():
+            paths = sorted(pathlib.Path('videos_example/').rglob('*.mp4'))
+            example_images = gr.Dataset(components=[video_input],
+                                    samples=[[path.as_posix()]
+                                             for path in paths if 'raw_videos'  in str(path)])
+
+    video_button.click(analize_video_serial, inputs=video_input, outputs=[video_output,image_temp])
+
+    example_images.click(fn=set_example_image,
+                         inputs=example_images,
+                         outputs=video_input)
+
+         
+demo.queue()
+if os.getenv('SYSTEM') == 'spaces':
+    demo.launch(width='40%',auth=(os.environ.get('SHARK_USERNAME'), os.environ.get('SHARK_PASSWORD')))
+else: 
+    demo.launch()

inference.py

@@ -15,10 +15,6 @@ from huggingface_hub import hf_hub_download
 from huggingface_hub import snapshot_download
 from time import time
 
-import concurrent.futures
-import threading
-
-
 classes= ['Beach',
  'Sea',
  'Wave',
@@ -73,23 +69,16 @@ classes= ['Beach',
  'Bull shark']*3
 
 
-
-
-
-
 REPO_ID = "SharkSpace/maskformer_model"
 FILENAME = "mask2former"
 
 snapshot_download(repo_id=REPO_ID, token= os.environ.get('SHARK_MODEL'),local_dir='model/')
 
-
-
-
 # Choose to use a config and initialize the detector
 config_file ='model/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco-panoptic/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco-panoptic.py'
 #'/content/mmdetection/configs/panoptic_fpn/panoptic-fpn_r50_fpn_ms-3x_coco.py'
 # Setup a checkpoint file to load
-checkpoint_file ='model/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco-panoptic/checkpoint.pth'
+checkpoint_file ='model/mask2former_swin-t-p4-w7-224_8xb2-lsj-50e_coco-panoptic/checkpoint_v2.pth'
 # '/content/drive/MyDrive/Algorithms/weights/shark_panoptic_weights_16_4_23/panoptic-fpn_r50_fpn_ms-3x_coco/epoch_36.pth'
 
 # register all modules in mmdet into the registries
@@ -106,7 +95,9 @@ print(dir(visualizer))
 # then pass to the model in init_detector
 visualizer.dataset_meta = model.dataset_meta
 def inference_frame_serial(image):
+    start = time()
     result = inference_detector(model, image)
+    print(f'inference time: {time()-start}')
     # show the results
     visualizer.add_datasample(
     'result',
@@ -118,7 +109,6 @@ def inference_frame_serial(image):
     frame = visualizer.get_image()
     return frame
 
-
 def inference_frame(image):
     result = inference_detector(model, image)
     # show the results