Resolved conflict in README.md

.gitignore

@@ -0,0 +1,50 @@
+data/
+data2/
+preds/
+preprocessed_data/
+wandb/
+
+.ruff_cache/
+
+*.sif
+*.log
+*.out
+
+.DS_Store
+
+__pycache__/
+*.py[cod]
+*$py.class
+
+*.so
+
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+.ipynb_checkpoints
+
+pip-log.txt
+pip-delete-this-directory.txt
+
+.python-version
+
+.env
+
+.mypy_cache
+

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Saruwatari&Saito laboratory, The University of Tokyo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,12 @@
 ---
 title: UTMOSv2
-emoji: 👁
-colorFrom: red
-colorTo: red
+emoji: 🌖
+colorFrom: yellow
+colorTo: green
 sdk: gradio
-sdk_version: 4.37.2
+sdk_version: 4.38.1
 app_file: app.py
 pinned: false
-license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,73 @@
+import importlib
+from types import SimpleNamespace
+
+import gradio as gr
+import pandas as pd
+
+# import spaces
+import torch
+
+from utmosv2.utils import get_dataset, get_model
+
+description = (
+    "# 🚀 UTMOSv2 demo\n\n"
+    "This is a demonstration of MOS prediction using UTMOSv2. "
+    "This demonstration only accepts `.wav` format. Best at 16 kHz sampling rate."
+)
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+config = importlib.import_module("utmosv2.config.fusion_stage3")
+cfg = SimpleNamespace(**{attr: getattr(config, attr) for attr in config.__dict__ if not attr.startswith("__")})
+cfg.reproduce = False
+cfg.config = "fusion_stage3"
+cfg.print_config = False
+cfg.data_config = None
+cfg.phase = "inference"
+cfg.weight = None
+cfg.num_workers = 1
+
+# @spaces.GPU
+def predict_mos(audio_path: str, domain: str) -> float:
+    data = pd.DataFrame({"file_path": [audio_path]})
+    data["dataset"] = domain
+    data['mos'] = 0
+
+    preds = 0.0
+    for fold in range(5):
+        cfg.now_fold = fold
+        model = get_model(cfg, device)
+        for _ in range(5):
+            test_dataset = get_dataset(cfg, data, "test")
+            p = model(*[torch.tensor(t).unsqueeze(0) for t in test_dataset[0][:-1]])
+            preds += p[0]
+    preds /= 25.0
+    return preds
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(description)
+    with gr.Row():
+        with gr.Column():
+            audio = gr.Audio(type="filepath", label="Audio")
+            domain = gr.Dropdown(
+                [
+                    "sarulab",
+                    "bvcc",
+                    "somos",
+                    "blizzard2008",
+                    "blizzard2009",
+                    "blizzard2010-EH1",
+                    "blizzard2010-EH2",
+                    "blizzard2010-ES1",
+                    "blizzard2010-ES3",
+                    "blizzard2011",
+                ],
+                label="Data-domain ID for the MOS prediction",
+            )
+            submit = gr.Button(value="Submit")
+
+        with gr.Column():
+            output = gr.Textbox(label="Predicted MOS", type="text")
+    submit.click(fn=predict_mos, inputs=[audio, domain], outputs=[output])
+
+demo.queue().launch()

models/fusion_stage3/fold0_s42_best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92fb17df6d4ef4708ea56cf0dca072b9b63d2b522190ef21ccee4f9ea80864fd
+size 818531314

models/fusion_stage3/fold1_s42_best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06756833fa743cb6683f109252ce178236464c33fbbee69d4e45cdf1ae7ad0cc
+size 818531314

models/fusion_stage3/fold2_s42_best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d784c0cdf9f4e4cbf697a8f755c8b0f5a0b842d18ad3d2bb42bbae3802d17a78
+size 818531314

models/fusion_stage3/fold3_s42_best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed133fe9b3e5cb78d0037aa784c7e23650a3e7b4f8ba00a73644c02aeb627758
+size 818531314

models/fusion_stage3/fold4_s42_best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12659b9db09d753654a7744b6b93ff6f90759a7429da793137b8cce107355967
+size 818531314

requirements.txt

@@ -0,0 +1,10 @@
+numpy>=1.24.4
+pandas>=2.2.2
+torch>=2.3.1
+timm>=1.0.7
+librosa>=0.10.2
+tqdm>=4.66.4
+scikit-learn>=1.3.2
+transformers>=4.42.4
+wandb>=0.17.0
+python-dotenv>=1.0.1

utmosv2/config/c_fusion_stage2.py

@@ -0,0 +1,149 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "sarulab"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="c_ssl_only_stage2",
+        spec_weight="c_spec_only_stage2",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/c_fusion_stage3.py

@@ -0,0 +1,149 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "sarulab"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="c_ssl_only_stage2",
+        spec_weight="c_spec_only_stage2",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/c_spec_only_stage1.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+external_data = []
+use_bvcc = True
+
+
+validation_dataset = "bvcc"
+
+dataset = SimpleNamespace(
+    name="multi_spec",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_specv2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/c_spec_only_stage2.py

@@ -0,0 +1,134 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+external_data = ["sarulab"]
+use_bvcc = False
+
+
+validation_dataset = "sarulab"
+
+dataset = SimpleNamespace(
+    name="multi_spec",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model = SimpleNamespace(
+    name="multi_specv2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/c_ssl_only_stage1.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "sarulab"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/c_ssl_only_stage2.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="stratified_group",
+    target="mos",
+    group="sys_id",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "sarulab"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/fusion_stage2.py

@@ -0,0 +1,151 @@
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path="preprocessed_data/clip_audio"
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = "all"
+use_bvcc = True
+
+predict_dataset = "ysaito"
+# predict_dataset = "bvcc"
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2",
+        spec_weight="spec_only",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path="preds",
+    submit_save_path="submissions",
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage2_wo_bc.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    # "blizzard2008",
+    # "blizzard2009",
+    # "blizzard2011",
+    # "blizzard2010-EH1",
+    # "blizzard2010-EH2",
+    # "blizzard2010-ES1",
+    # "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_bc",
+        spec_weight="spec_only_wo_bc",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage2_wo_bvcc.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = False
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_bvcc",
+        spec_weight="spec_only_wo_bvcc",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage2_wo_sarulab.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    # "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_sarulab",
+        spec_weight="spec_only_wo_sarulab",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage2_wo_somos.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 16
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    # "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-5)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_somos",
+        spec_weight="spec_only_wo_somos",
+        num_classes=1,
+        freeze=True,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=8,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage3.py

@@ -0,0 +1,150 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2",
+        spec_weight="spec_only",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage3_wo_bc.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    # "blizzard2008",
+    # "blizzard2009",
+    # "blizzard2011",
+    # "blizzard2010-EH1",
+    # "blizzard2010-EH2",
+    # "blizzard2010-ES1",
+    # "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_bc",
+        spec_weight="spec_only_wo_bc",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage3_wo_bvcc.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = False
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_bvcc",
+        spec_weight="spec_only_wo_bvcc",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage3_wo_sarulab.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    # "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_sarulab",
+        spec_weight="spec_only_wo_sarulab",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_stage3_wo_somos.py

@@ -0,0 +1,160 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    # "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=5e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-8)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2_wo_somos",
+        spec_weight="spec_only_wo_somos",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=2,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_wo_stage1and2.py

@@ -0,0 +1,150 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight=None,
+        spec_weight=None,
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/fusion_wo_stage2.py

@@ -0,0 +1,150 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 8
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="ssl_multispec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-4, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="ssl_multispec_ext_v2",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+    ssl_spec=SimpleNamespace(
+        ssl_weight="ssl_only_stage2",
+        spec_weight="spec_only",
+        num_classes=1,
+        freeze=False,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/spec_only.py

@@ -0,0 +1,135 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="multi_spec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_spec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/spec_only_wo_bc.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    # "blizzard2008",
+    # "blizzard2009",
+    # "blizzard2011",
+    # "blizzard2010-EH1",
+    # "blizzard2010-EH2",
+    # "blizzard2010-ES1",
+    # "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="multi_spec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_spec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/spec_only_wo_bvcc.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = False
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="multi_spec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_spec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/spec_only_wo_sarulab.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    # "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="multi_spec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_spec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/spec_only_wo_somos.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+from torchvision import transforms
+
+from utmosv2.transform.xymasking import XYMasking
+
+batch_size = 10
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    # "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+dataset = SimpleNamespace(
+    name="multi_spec_ext",
+    specs=[
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=4096,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=2048,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=1024,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+        SimpleNamespace(
+            mode="melspec",
+            n_fft=4096,
+            hop_length=32,
+            win_length=512,
+            n_mels=512,
+            shape=(512, 512),
+            norm=80,
+        ),
+    ],
+    spec_frames=SimpleNamespace(
+        num_frames=2, frame_sec=1.4, mixup_inner=True, mixup_alpha=0.4, extend="tile"
+    ),
+)
+transform = dict(
+    train=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            XYMasking(
+                num_masks_x=(0, 2),
+                num_masks_y=(0, 2),
+                mask_x_length=(10, 40),
+                mask_y_length=(10, 30),
+                fill_value=0,
+                p=0.5,
+            ),
+            # transforms.ToTensor(),
+        ]
+    ),
+    valid=transforms.Compose(
+        [
+            transforms.Resize((512, 512)),
+            # transforms.ToTensor()
+        ]
+    ),
+)
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model = SimpleNamespace(
+    name="multi_spec_ext",
+    multi_spec=SimpleNamespace(
+        backbone="tf_efficientnetv2_s.in21k_ft_in1k",
+        pretrained=True,
+        num_classes=1,
+        pool_type="catavgmax",
+        # feature_height=16,
+        atten=True,
+        # classifier=None,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    extend="tile",
+)

utmosv2/config/ssl_only_stage1.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage1_wo_bc.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    # "blizzard2008",
+    # "blizzard2009",
+    # "blizzard2011",
+    # "blizzard2010-EH1",
+    # "blizzard2010-EH2",
+    # "blizzard2010-ES1",
+    # "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage1_wo_bvcc.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = False
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage1_wo_sarulab.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    # "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage1_wo_somos.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    # "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=1e-3, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-7)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=True,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=20,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage2.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = "all"
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage2_wo_bc.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    # "blizzard2008",
+    # "blizzard2009",
+    # "blizzard2011",
+    # "blizzard2010-EH1",
+    # "blizzard2010-EH2",
+    # "blizzard2010-ES1",
+    # "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage2_wo_bvcc.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = False
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage2_wo_sarulab.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    # "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/config/ssl_only_stage2_wo_somos.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+batch_size = 32
+num_folds = 5
+
+sr = 16000
+
+preprocess = SimpleNamespace(
+    top_db=30, min_seconds=None, save_path=Path("preprocessed_data")
+)
+
+split = SimpleNamespace(
+    type="sgkf_kind",
+    target="mos",
+    group="sys_id",
+    kind="dataset",
+)
+
+dataset = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        duration=3,
+    ),
+)
+
+external_data = [
+    "sarulab",
+    "blizzard2008",
+    "blizzard2009",
+    "blizzard2011",
+    "blizzard2010-EH1",
+    "blizzard2010-EH2",
+    "blizzard2010-ES1",
+    "blizzard2010-ES3",
+    # "somos",
+]
+use_bvcc = True
+
+
+validation_dataset = "each"
+
+loss = [
+    (SimpleNamespace(name="pairwize_diff", margin=0.2, norm="l1"), 0.7),
+    (SimpleNamespace(name="mse"), 0.2),
+]
+
+optimizer = SimpleNamespace(name="adamw", lr=3e-5, weight_decay=1e-4)
+
+scheduler = SimpleNamespace(name="cosine", T_max=None, eta_min=1e-9)
+
+model_path = "model"
+model = SimpleNamespace(
+    name="sslext",
+    ssl=SimpleNamespace(
+        name="facebook/wav2vec2-base",
+        attn=1,
+        freeze=False,
+        num_classes=1,
+    ),
+)
+
+run = SimpleNamespace(
+    mixup=True,
+    mixup_alpha=0.4,
+    num_epochs=5,
+)
+
+main_metric = "sys_srcc"
+id_name = None
+
+
+inference = SimpleNamespace(
+    save_path=Path("preds"),
+    submit_save_path=Path("submissions"),
+    num_tta=5,
+    batch_size=8,
+    # extend="tile",
+)

utmosv2/dataset/__init__.py

@@ -0,0 +1,11 @@
+from utmosv2.dataset.multi_spec import MultiSpecDataset, MultiSpecExtDataset
+from utmosv2.dataset.ssl import SSLDataset, SSLExtDataset
+from utmosv2.dataset.ssl_multispec import SSLLMultiSpecExtDataset
+
+__all__ = [
+    "MultiSpecDataset",
+    "MultiSpecExtDataset",
+    "SSLLMultiSpecExtDataset",
+    "SSLDataset",
+    "SSLExtDataset",
+]

utmosv2/dataset/_utils.py

@@ -0,0 +1,53 @@
+import json
+
+import librosa
+import numpy as np
+
+
+def load_audio(cfg, file: str) -> np.ndarray:
+    if file.endswith(".wav"):
+        y, sr = librosa.load(file, sr=None)
+        y = librosa.resample(y, orig_sr=sr, target_sr=cfg.sr)
+    else:
+        y = np.load(file)
+    return y
+
+
+def extend_audio(cfg, y: np.ndarray, length: int, type: str) -> np.ndarray:
+    if y.shape[0] > length:
+        return y
+    elif type == "tile":
+        n = length // y.shape[0] + 1
+        y = np.tile(y, n)
+        return y
+    else:
+        raise NotImplementedError
+
+
+def select_random_start(y: np.ndarray, length: int) -> np.ndarray:
+    start = np.random.randint(0, y.shape[0] - length)
+    return y[start : start + length]
+
+
+def get_dataset_map(cfg):
+    if cfg.data_config:
+        with open(cfg.data_config, "r") as f:
+            datasets = json.load(f)
+        return {d["name"]: i for i, d in enumerate(datasets["data"])}
+    else:
+        return {
+            "bvcc": 0,
+            "sarulab": 1,
+            "blizzard2008": 2,
+            "blizzard2009": 3,
+            "blizzard2010-EH1": 4,
+            "blizzard2010-EH2": 5,
+            "blizzard2010-ES1": 6,
+            "blizzard2010-ES3": 7,
+            "blizzard2011": 8,
+            "somos": 9,
+        }
+
+
+def get_dataset_num(cfg):
+    return len(get_dataset_map(cfg))

utmosv2/dataset/multi_spec.py

@@ -0,0 +1,99 @@
+import librosa
+import numpy as np
+import pandas as pd
+import torch
+
+from utmosv2.dataset._utils import (
+    extend_audio,
+    get_dataset_map,
+    load_audio,
+    select_random_start,
+)
+
+
+class MultiSpecDataset(torch.utils.data.Dataset):
+    def __init__(self, cfg, data: pd.DataFrame, phase: str, transform=None):
+        self.cfg = cfg
+        self.data = data
+        self.phase = phase
+        self.transform = transform
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        row = self.data.iloc[idx]
+        file = row["file_path"]
+        y = load_audio(self.cfg, file)
+        specs = []
+        length = int(self.cfg.dataset.spec_frames.frame_sec * self.cfg.sr)
+        y = extend_audio(self.cfg, y, length, type=self.cfg.dataset.spec_frames.extend)
+        for _ in range(self.cfg.dataset.spec_frames.num_frames):
+            y1 = select_random_start(y, length)
+            for spec_cfg in self.cfg.dataset.specs:
+                spec = _make_spctrogram(self.cfg, spec_cfg, y1)
+                if self.cfg.dataset.spec_frames.mixup_inner:
+                    y2 = select_random_start(y, length)
+                    spec2 = _make_spctrogram(self.cfg, spec_cfg, y2)
+                    lmd = np.random.beta(
+                        self.cfg.dataset.spec_frames.mixup_alpha,
+                        self.cfg.dataset.spec_frames.mixup_alpha,
+                    )
+                    spec = lmd * spec + (1 - lmd) * spec2
+                spec = np.stack([spec, spec, spec], axis=0)
+                # spec = np.transpose(spec, (1, 2, 0))
+                spec = torch.tensor(spec, dtype=torch.float32)
+                phase = "train" if self.phase == "train" else "valid"
+                spec = self.transform[phase](spec)
+                specs.append(spec)
+        spec = torch.stack(specs).float()
+
+        target = row["mos"]
+        target = torch.tensor(target, dtype=torch.float32)
+
+        return spec, target
+
+
+class MultiSpecExtDataset(MultiSpecDataset):
+    def __init__(self, cfg, data: pd.DataFrame, phase: str, transform=None):
+        super().__init__(cfg, data, phase, transform)
+        self.dataset_map = get_dataset_map(cfg)
+
+    def __getitem__(self, idx):
+        spec, target = super().__getitem__(idx)
+
+        d = np.zeros(len(self.dataset_map))
+        d[self.dataset_map[self.data.iloc[idx]["dataset"]]] = 1
+        d = torch.tensor(d, dtype=torch.float32)
+
+        return spec, d, target
+
+
+def _make_spctrogram(cfg, spec_cfg, y: np.ndarray) -> np.ndarray:
+    if spec_cfg.mode == "melspec":
+        return _make_melspec(cfg, spec_cfg, y)
+    elif spec_cfg.mode == "stft":
+        return _make_stft(cfg, spec_cfg, y)
+    else:
+        raise NotImplementedError
+
+
+def _make_melspec(cfg, spec_cfg, y: np.ndarray) -> np.ndarray:
+    spec = librosa.feature.melspectrogram(
+        y=y,
+        sr=cfg.sr,
+        n_fft=spec_cfg.n_fft,
+        hop_length=spec_cfg.hop_length,
+        n_mels=spec_cfg.n_mels,
+    )
+    spec = librosa.power_to_db(spec, ref=np.max)
+    if spec_cfg.norm is not None:
+        spec = (spec + spec_cfg.norm) / spec_cfg.norm
+    return spec
+
+
+def _make_stft(cfg, spec_cfg, y: np.ndarray) -> np.ndarray:
+    spec = librosa.stft(y=y, n_fft=spec_cfg.n_fft, hop_length=spec_cfg.hop_length)
+    spec = np.abs(spec)
+    spec = librosa.amplitude_to_db(spec)
+    return spec

utmosv2/dataset/ssl.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+import torch
+
+from utmosv2.dataset._utils import (
+    extend_audio,
+    get_dataset_map,
+    load_audio,
+    select_random_start,
+)
+
+
+class SSLDataset(torch.utils.data.Dataset):
+    def __init__(self, cfg, data: pd.DataFrame, phase: str):
+        self.cfg = cfg
+        self.data = data
+        self.phase = phase
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        row = self.data.iloc[idx]
+        file = row["file_path"]
+        y = load_audio(self.cfg, file)
+        length = int(self.cfg.dataset.ssl.duration * self.cfg.sr)
+        y = extend_audio(self.cfg, y, length, type="tile")
+        y = select_random_start(y, length)
+
+        target = row["mos"]
+        target = torch.tensor(target, dtype=torch.float32)
+
+        return y, target
+
+
+class SSLExtDataset(SSLDataset):
+    def __init__(self, cfg, data: pd.DataFrame, phase: str):
+        super().__init__(cfg, data, phase)
+        self.dataset_map = get_dataset_map(cfg)
+
+    def __getitem__(self, idx):
+        y, target = super().__getitem__(idx)
+
+        d = np.zeros(len(self.dataset_map))
+        d[self.dataset_map[self.data.iloc[idx]["dataset"]]] = 1
+        d = torch.tensor(d, dtype=torch.float32)
+
+        return y, d, target

utmosv2/dataset/ssl_multispec.py

@@ -0,0 +1,20 @@
+import pandas as pd
+import torch
+
+from utmosv2.dataset import MultiSpecDataset, SSLExtDataset
+
+
+class SSLLMultiSpecExtDataset(torch.utils.data.Dataset):
+    def __init__(self, cfg, data: pd.DataFrame, phase: str, transform=None):
+        self.data = data
+        self.ssl = SSLExtDataset(cfg, data, phase)
+        self.multi_spec = MultiSpecDataset(cfg, data, phase, transform)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        x1, d, target = self.ssl[idx]
+        x2, _ = self.multi_spec[idx]
+
+        return x1, x2, d, target

utmosv2/loss/__init__.py

@@ -0,0 +1,3 @@
+from utmosv2.loss.losses import CombinedLoss, PairwizeDiffLoss
+
+__all__ = ["PairwizeDiffLoss", "CombinedLoss"]

utmosv2/loss/losses.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class PairwizeDiffLoss(nn.Module):
+    def __init__(self, margin: float = 0.2, norm: str = "l1"):
+        super().__init__()
+        self.margin = margin
+        self.norm = norm
+
+    def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+        s = input.unsqueeze(1) - input.unsqueeze(0)
+        t = target.unsqueeze(1) - target.unsqueeze(0)
+        if self.norm not in ["l1", "l2_squared"]:
+            raise ValueError(
+                f'Unknown norm: {self.norm}. Must be one of ["l1", "l2_squared"]'
+            )
+        norm_fn = {
+            "l1": torch.abs,
+            "l2_squared": lambda x: x**2,
+        }[self.norm]
+        loss = F.relu(norm_fn(s - t) - self.margin)
+        return loss.mean().div(2)
+
+
+class CombinedLoss(nn.Module):
+    def __init__(self, weighted_losses: list[tuple[nn.Module, float]]):
+        super().__init__()
+        self.weighted_losses = weighted_losses
+
+    def forward(
+        self, input: torch.Tensor, target: torch.Tensor
+    ) -> list[tuple[float, torch.Tensor]]:
+        return [(w, loss(input, target)) for loss, w in self.weighted_losses]