pipeline/process_videos.py

import base64

import cv2
import pandas as pd
from PIL import Image
from tqdm import tqdm

from pipeline.clip_wrapper import MODEL_DIM, ClipWrapper
from pipeline.download_videos import DATA_DIR, REPO_ROOT, VIDEO_DIR

FRAME_EXTRACT_RATE_SECONDS = 5  # Extract a frame every 5 seconds
IMAGES_DIR = DATA_DIR / "images"
DATAFRAME_PATH = DATA_DIR / "dataset.parquet"


def process_videos() -> None:
    IMAGES_DIR.mkdir(exist_ok=True, parents=True)
    (IMAGES_DIR / ".gitignore").write_text("*")
    "Runs clip on video frames, saves results to a parquet file"
    clip_wrapper = ClipWrapper()
    results = []
    for video_path in tqdm(list(VIDEO_DIR.glob("*.mp4")), desc="Processing videos"):
        video_id = video_path.stem
        extracted_images_dir = IMAGES_DIR / video_id
        extracted_images_dir.mkdir(exist_ok=True, parents=True)
        complete_file = extracted_images_dir / "complete"
        if complete_file.exists():
            continue
        for clip_vector, image, timestamp_secs, frame_idx in get_clip_vectors(
            video_path, clip_wrapper
        ):
            image_path = extracted_images_dir / f"{frame_idx}.jpg"
            image.save(image_path)
            with open(image_path, "rb") as f:
                encoded_image = base64.b64encode(f.read()).decode()
            results.append(
                [
                    video_id,
                    frame_idx,
                    timestamp_secs,
                    encoded_image,
                    *clip_vector,
                ]
            )
        complete_file.touch()
    df = pd.DataFrame(
        results,
        columns=["video_id", "frame_idx", "timestamp", "base64_image"]
        + [f"dim_{i}" for i in range(MODEL_DIM)],
    )
    print(f"Saving data to {DATAFRAME_PATH}")
    df.to_parquet(DATAFRAME_PATH, index=False)


def get_clip_vectors(video_path, clip_wrapper):
    cap = cv2.VideoCapture(str(video_path))
    num_video_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = int(cap.get(cv2.CAP_PROP_FPS))
    extract_every_n_frames = FRAME_EXTRACT_RATE_SECONDS * fps
    for frame_idx in tqdm(range(num_video_frames), desc="Running CLIP on video"):
        ret, frame = cap.read()
        if frame_idx % extract_every_n_frames != 0:
            continue
        image = Image.fromarray(frame[..., ::-1])
        clip_vector = clip_wrapper.images2vec([image]).squeeze().numpy()
        timestamp_secs = frame_idx / fps
        yield clip_vector, image, timestamp_secs, frame_idx
    cap.release()


if __name__ == "__main__":
    process_videos()