lotta changes

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

.zshrc

@@ -8,6 +8,19 @@
 #    - conda-env: Adds support for Conda environment management
 # 4. Set the custom theme for the shell prompt
 
+#display_git_help() {
+#  echo "Git"
+#  echo "---"
+#  echo "ga: \`git add . && git commit -avs --verbose && git push\`"
+#  echo "gc: \`git commit -avs --verbose\`"
+#  echo "gcs: \`git clone --recurse-submodules\`"
+#  echo "grh: \`git reset --hard\`"
+#  echo "wd: \`git diff --word-diff-regex='[^,]+' --patience\`"
+#  echo "gs: \`git status\`"
+#  echo "gcx: \`git clean -fxd\`"
+#}
+#display_git_help
+
 # The `export QT_QPA_PLATFORM=offscreen` command is used to set the `QT_QPA_PLATFORM`
 # environment variable to `offscreen`. This is particularly useful when running Qt
 # applications in a headless environment, such as a server or a CI/CD pipeline,
@@ -19,11 +32,11 @@ export QT_QPA_PLATFORM=offscreen
 
 # Enable the experimental Just-In-Time (JIT) compiler for Python 3.13.
 # This can improve performance by compiling Python code to machine code at runtime.
-# Note: The JIT is only available for x86_64 builds of Python in conda[^1^][1].
+# Note: The JIT is only available for x86_64 builds of Python in conda.
 export PYTHON_JIT=1
 
 # Load the custom git wrapper script
-source $HOME/toolkit/git-wrapper.zsh
+source $HOME/toolkit/zsh/git-wrapper.zsh
 
 # Set the path to the Oh My Zsh installation directory
 export ZSH="$HOME/.oh-my-zsh"
@@ -204,19 +217,6 @@ alias rl="source ~/.zshrc"
 # Alias for quickly editing and reloading the zsh configuration file
 alias ezc="nvim ~/.zshrc && source ~/.zshrc"
 
-display_git_help() {
-  echo "Git"
-  echo "---"
-  echo "ga: \`git add . && git commit -avs --verbose && git push\`"
-  echo "gc: \`git commit -avs --verbose\`"
-  echo "gcs: \`git clone --recurse-submodules\`"
-  echo "grh: \`git reset --hard\`"
-  echo "wd: \`git diff --word-diff-regex='[^,]+' --patience\`"
-  echo "gs: \`git status\`"
-  echo "gcx: \`git clean -fxd\`"
-}
-display_git_help
-
 # This function copies the sample prompts file to each dataset directory.
 # It iterates through all directories in ~/datasets that start with "by_"
 # and copies the kade-sample-prompts.txt file from the toolkit directory
@@ -1047,6 +1047,9 @@ conda_prompt_info() {
   fi
 }
 
+# Setup zoxide
+eval "$(zoxide init zsh)"
+
 # Function: display_custom_help
 # Description:
 #   This function displays custom help information for user-defined functions and aliases.
@@ -1060,4 +1063,5 @@ conda_prompt_info() {
 #
 # Note:
 #   Add or modify entries in this function to keep your personal command reference up-to-date.
-display_custom_help
+#display_custom_help
+

cringe_move_exaple_images.py

@@ -0,0 +1,33 @@
+import os
+import re
+
+def move_example_images_section(file_path):
+    with open(file_path, 'r') as file:
+        content = file.read()
+
+    example_images_section = re.search(r'## Example Images.*?(?=##|$)', content, re.DOTALL)
+    introduction_index = content.find('## Introduction')
+
+    if example_images_section and introduction_index != -1:
+        example_images_text = example_images_section.group()
+
+        # Remove the "## Example Images" section from its original position
+        content = content.replace(example_images_text, '')
+
+        # Insert the "## Example Images" section above the "## Introduction" section
+        new_content = content[:introduction_index] + example_images_text + '\n' + content[introduction_index:]
+
+        with open(file_path, 'w') as file:
+            file.write(new_content)
+
+def recurse_directories(target_dir='.'):
+    for root, dirs, files in os.walk(target_dir):
+        for file in files:
+            if file.endswith('.md'):
+                file_path = os.path.join(root, file)
+                move_example_images_section(file_path)
+
+if __name__ == '__main__':
+    target_directory = '.'  # Default to current directory
+    recurse_directories(target_directory)
+

move_example_section

@@ -0,0 +1,33 @@
+import os
+
+def add_example_images_section(file_path):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    introduction_index = None
+    for i, line in enumerate(lines):
+        if line.strip().startswith('## Introduction'):
+            introduction_index = i
+            break
+
+    if introduction_index is not None:
+        example_images_section = '## Example Images\n\n'
+        if example_images_section not in lines:
+            lines.insert(introduction_index, example_images_section)
+
+    with open(file_path, 'w') as file:
+        file.writelines(lines)
+
+def recurse_directories(target_dir='.'):
+    for root, dirs, files in os.walk(target_dir):
+        for file in files:
+            if file.endswith('.md'):
+                file_path = os.path.join(root, file)
+                add_example_images_section(file_path)
+
+if __name__ == '__main__':
+    target_directory = input('Enter the target directory (leave blank for current directory): ')
+    if not target_directory:
+        target_directory = '.'
+    recurse_directories(target_directory)
+

select_and_concat_captions

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""
+This script walks through a directory, identifies image files, and checks for the existence of corresponding
+.caption and .tags files. It then concatenates the contents of .caption and .tags files into the .txt files.
+
+Usage:
+    - Place the script in the directory containing the image files.
+    - Run the script to concatenate .caption and .tags files into .txt files.
+    - Use the dry_run flag to preview the changes without writing to the .txt files.
+
+Functions:
+    get_files(path): Walks through the directory and yields image files along with their .caption and .tags files.
+    concat(caption_path, tags_path, txt_path, dry_run=False): Concatenates the contents of .caption and .tags files into the .txt file.
+"""
+from pathlib import Path
+import os
+import re
+
+FILE_EXTS = {".png", ".jpg", ".jpeg", ".tiff", ".bmp", ".gif", ".jxl"}
+
+def get_files(path):
+    path = Path(path)
+    # Walk the directory, looking for image files
+    for root, dirs, files in os.walk(path):
+        root = path / root
+        for file in files:
+            file = root / file
+            if file.suffix not in FILE_EXTS:
+                continue
+            caption = file.with_suffix(".caption")
+            tags = file.with_suffix(".tags")
+            txt = file.with_suffix(".txt")
+            if not caption.exists():
+                print(f"{caption} does not exist")
+            if not tags.exists():
+                print(f"{tags} does not exist")
+            yield file, caption, tags, txt
+
+def select_best_caption(caption_path):
+    with open(caption_path, "r") as f:
+        content = f.read().strip()
+    
+    captions = re.split(r'----------', content)
+    captions = [caption.replace('\n', ' ').strip() for caption in captions if caption.strip()]
+    
+    best_caption = ""
+    for caption in captions:
+        if caption and caption[-1] in ".!?":
+            if len(caption) > len(best_caption):
+                best_caption = caption
+    
+    return best_caption
+
+def concat(caption_path, tags_path, txt_path, dry_run=False):
+    best_caption = select_best_caption(caption_path)
+    if not best_caption:
+        print(f"No suitable caption found in {caption_path}")
+        return
+    
+    with open(tags_path, "r") as f:
+        tags = f.read().strip(", \n")
+    
+    txt = f"{tags}, {best_caption}"
+    
+    if dry_run:
+        print(f"{txt_path}:")
+        print(txt)
+        print()
+    else:
+        with open(txt_path, 'w') as f:
+            f.write(txt)
+        print(f"wrote {txt_path}")
+
+if __name__ == "__main__":
+    dry_run = False
+    for f in get_files("."):
+        concat(*f[1:], dry_run=dry_run)
+

txt2tags

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+import os
+from pathlib import Path
+
+def rename_txt_to_tags(target_dir='.'):
+    for root, dirs, files in os.walk(target_dir):
+        for file in files:
+            if file.endswith('.txt'):
+                txt_file = Path(root) / file
+                tags_file = txt_file.with_suffix('.tags')
+                os.rename(txt_file, tags_file)
+                print(f"Renamed {txt_file} to {tags_file}")
+
+if __name__ == '__main__':
+    target_directory = input('Enter the target directory (leave blank for current directory): ')
+    if not target_directory:
+        target_directory = '.'
+    rename_txt_to_tags(target_directory)
+

videocaption

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import io
+
+import argparse
+import numpy as np
+import torch
+from decord import cpu, VideoReader, bridge
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL_PATH = "THUDM/cogvlm2-llama3-caption"
+
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+TORCH_TYPE = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.get_device_capability()[
+    0] >= 8 else torch.float16
+
+parser = argparse.ArgumentParser(description="CogVLM2-Video CLI Demo")
+parser.add_argument('--quant', type=int, choices=[4, 8], help='Enable 4-bit or 8-bit precision loading', default=0)
+args = parser.parse_args([])
+
+
+def load_video(video_data, strategy='chat'):
+    bridge.set_bridge('torch')
+    mp4_stream = video_data
+    num_frames = 24
+    decord_vr = VideoReader(io.BytesIO(mp4_stream), ctx=cpu(0))
+
+    frame_id_list = None
+    total_frames = len(decord_vr)
+    if strategy == 'base':
+        clip_end_sec = 60
+        clip_start_sec = 0
+        start_frame = int(clip_start_sec * decord_vr.get_avg_fps())
+        end_frame = min(total_frames,
+                        int(clip_end_sec * decord_vr.get_avg_fps())) if clip_end_sec is not None else total_frames
+        frame_id_list = np.linspace(start_frame, end_frame - 1, num_frames, dtype=int)
+    elif strategy == 'chat':
+        timestamps = decord_vr.get_frame_timestamp(np.arange(total_frames))
+        timestamps = [i[0] for i in timestamps]
+        max_second = round(max(timestamps)) + 1
+        frame_id_list = []
+        for second in range(max_second):
+            closest_num = min(timestamps, key=lambda x: abs(x - second))
+            index = timestamps.index(closest_num)
+            frame_id_list.append(index)
+            if len(frame_id_list) >= num_frames:
+                break
+
+    video_data = decord_vr.get_batch(frame_id_list)
+    video_data = video_data.permute(3, 0, 1, 2)
+    return video_data
+
+
+tokenizer = AutoTokenizer.from_pretrained(
+    MODEL_PATH,
+    trust_remote_code=True,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_PATH,
+    torch_dtype=TORCH_TYPE,
+    trust_remote_code=True
+).eval().to(DEVICE)
+
+
+def predict(prompt, video_data, temperature):
+    strategy = 'chat'
+
+    video = load_video(video_data, strategy=strategy)
+
+    history = []
+    query = prompt
+    inputs = model.build_conversation_input_ids(
+        tokenizer=tokenizer,
+        query=query,
+        images=[video],
+        history=history,
+        template_version=strategy
+    )
+    inputs = {
+        'input_ids': inputs['input_ids'].unsqueeze(0).to('cuda'),
+        'token_type_ids': inputs['token_type_ids'].unsqueeze(0).to('cuda'),
+        'attention_mask': inputs['attention_mask'].unsqueeze(0).to('cuda'),
+        'images': [[inputs['images'][0].to('cuda').to(TORCH_TYPE)]],
+    }
+    gen_kwargs = {
+        "max_new_tokens": 2048,
+        "pad_token_id": 128002,
+        "top_k": 1,
+        "do_sample": False,
+        "top_p": 0.1,
+        "temperature": temperature,
+    }
+    with torch.no_grad():
+        outputs = model.generate(**inputs, **gen_kwargs)
+        outputs = outputs[:, inputs['input_ids'].shape[1]:]
+        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return response
+
+
+def test():
+    prompt = "Please describe this video in detail."
+    temperature = 0.1
+    video_data = open('test.mp4', 'rb').read()
+    response = predict(prompt, video_data, temperature)
+    print(response)
+
+
+if __name__ == '__main__':
+    test()
+

zsh/png2mp4.zsh

@@ -48,8 +48,8 @@ png2mp4() {
             vf_options="select='not(mod(n\,$max_images))',$vf_options"
         fi
 
-        ffmpeg -framerate 60 -pattern_type glob -i "$temp_dir/*_${sample}_*.png" -vf "$vf_options" -crf 28 \
-        -c:v libx264 -pix_fmt yuv420p -y "$temp_dir/temp_${sample}.mp4"
+        ffmpeg -framerate 60 -pattern_type glob -i "$temp_dir/*_${sample}_*.png" -vf "$vf_options" -crf 18 \
+        -c:v libx264 -b:v 12M -pix_fmt yuv420p -y "$temp_dir/temp_${sample}.mp4"
 
         if [ $? -ne 0 ]; then
             echo "Error: ffmpeg command failed for sample ${sample}."
@@ -60,7 +60,7 @@ png2mp4() {
         echo "Processing final video for sample ${sample}..."
         duration=$(ffmpeg -i "$temp_dir/temp_${sample}.mp4" 2>&1 | grep 'Duration' | awk '{print $2}' | tr -d , | awk -F: '{print ($1 * 3600) + ($2 * 60) + $3}')
         fade_start=$(echo "$duration + 3" | bc)
-        ffmpeg -i "$temp_dir/temp_${sample}.mp4" -vf "tpad=stop_mode=clone:stop_duration=8,fade=t=out:st=$fade_start:d=5" -c:v libx264 -pix_fmt yuv420p -y "$output_filename"
+        ffmpeg -i "$temp_dir/temp_${sample}.mp4" -vf "tpad=stop_mode=clone:stop_duration=8,fade=t=out:st=$fade_start:d=5" -c:v libx264 -b:v 12M -crf 18 -pix_fmt yuv420p -y "$output_filename"
 
         if [ $? -ne 0 ]; then
             echo "Error: Final ffmpeg processing failed for sample ${sample}."