lots of random changes

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jxl filter=lfs diff=lfs merge=lfs -text

.zshrc

@@ -1,3 +1,5 @@
+source $HOME/toolkit/git-wrapper.zsh
+
 export ZSH="$HOME/.oh-my-zsh"
 
 plugins=(git autojump conda-env)
@@ -38,6 +40,7 @@ export PATH=$PATH:$HOME/source/repos/dataset-tools/target/x86_64-unknown-linux-g
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib:/opt/cuda/targets/x86_64-linux/lib
 export COMFYUI_MODEL_PATH=/home/kade/ComfyUI/models
 
+alias upx='/home/kade/.local/bin/upx'
 alias ls='ls --color=always'
 alias ll="ls -lah --color=always"
 alias cp='cp --reflink=auto'

crawl/crawl

@@ -124,8 +124,8 @@ def save_result(target_url):
     # Choose the appropriate base path based on the operating system
     if platform.system() == "Windows":
         base_path = "E:\\datasets\\knowledgebase\\Saved Websites\\"
-    else:
-        base_path = "/home/kade/saved_websites/"
+    elsek:
+        base_path = "/home/kade/datasets/knowledgebase/Saved Websites"
 
     save_dir = os.path.join(base_path, sanitized_title)
     os.makedirs(save_dir, exist_ok=True)

git-wrapper.zsh

@@ -0,0 +1,46 @@
+# This function is a wrapper for the 'git' command, providing additional functionality for the 'clone' and 'add' operations.
+# 
+# Usage:
+#   git <command> [<args>...]
+#
+# Parameters:
+#   <command> - The git command to execute (e.g., clone, pull, push, add, etc.).
+#   [<args>...] - The arguments to pass to the git command.
+#
+# Functionality:
+#   - If the first argument is 'clone', the function executes 'git clone' with the '--recurse-submodules' option.
+#     This ensures that all submodules are cloned along with the main repository.
+#   - If the first argument is 'add', the function checks if the current directory is inside '~/datasets'.
+#     If it is, it checks for the existence of a '.gitattributes' file and runs 'git lfs track "*.jxl"' before executing 'git add'.
+#   - For any other git command, the function simply passes all arguments to the 'git' command as-is.
+#
+# Examples:
+#   git clone https://github.com/user/repo.git
+#     This will execute 'git clone --recurse-submodules https://github.com/user/repo.git', cloning the repository along with its submodules.
+#
+#   git add .
+#     This will check if the current directory is inside '~/datasets', check for a '.gitattributes' file, and run 'git lfs track "*.jxl"' if the file exists, then execute 'git add .'.
+#
+#   git pull origin main
+#     This will execute 'git pull origin main', pulling the latest changes from the 'main' branch.
+#
+# Notes:
+#   - This function uses 'command' to bypass any other shell functions or aliases named 'git', ensuring that the actual git command is executed.
+#   - The use of "${@:2}" in the 'clone' case ensures that all arguments after 'clone' are passed to the 'git clone' command.
+#   - The use of "${@:2}" in the 'add' case ensures that all arguments after 'add' are passed to the 'git add' command.
+#   - The function checks if the current directory is inside '~/datasets' by comparing the current directory path with the '~/datasets' path.
+function git() {
+    if [[ "$1" == "clone" ]]; then
+        command git clone --recurse-submodules "${@:2}"
+    elif [[ "$1" == "add" ]]; then
+        if [[ "$(pwd)" == ~/datasets* ]]; then
+            if [[ -f .gitattributes ]]; then
+                command git lfs track "*.jxl"
+            fi
+        fi
+        command git add "${@:2}"
+    else
+        command git "$@"
+    fi
+}
+