Upload folder using huggingface_hub

.gitattributes

@@ -34,3 +34,25 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 mucodec.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/1.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/2.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/3.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/4.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_1.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_2.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_3.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_4.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_5.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/en/prompt_6.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_2.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_3.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_4.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_5.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/audio_prompt/zh/prompt_6.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/142001_0_rlhf.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/142023_0_rlhf.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/142038_0_rlhf.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/64474_1_rlhf.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/demo_data/142038_0_rlhf.wav filter=lfs diff=lfs merge=lfs -text
+Wav-Omni/examples/s2s/demo/question_jsp.wav filter=lfs diff=lfs merge=lfs -text

Wav-Omni/.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,79 @@
+name: 🐛 Bug Report
+description: Create a report to help us reproduce and fix the bug
+
+body:
+  - type: markdown
+    attributes:
+      value: >
+        #### Before submitting a bug, please make sure the issue hasn't been already addressed by searching through [the
+        existing and past issues](https://github.com/ddlBoJack/Wav-Omni/issues).
+
+  - type: textarea
+    id: system-info
+    attributes:
+      label: System Info
+      description: |
+        Please share your system info with us. You can use the following command to capture your environment information
+        python -m "torch.utils.collect_env"
+
+      placeholder: | 
+        PyTorch version, CUDA version, GPU type, #num of GPUs...   
+    validations:
+      required: true
+
+  - type: checkboxes
+    id: information-scripts-examples
+    attributes:
+      label: Information
+      description: 'The problem arises when using:'
+      options:
+        - label: "The official example scripts"
+        - label: "My own modified scripts"
+
+  - type: textarea
+    id: bug-description
+    attributes:
+      label: 🐛 Describe the bug
+      description: |
+        Please provide a clear and concise description of what the bug is.
+
+        Provide the exact command(s) that you ran with the settings eg using FSDP and PEFT or pure FSDP.
+        
+        Please also paste or describe the results you observe instead of the expected results. 
+      placeholder: |
+        A clear and concise description of what the bug is.
+        
+        ```python
+        # Command that you used for running the examples
+        ```
+        Description of the results
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Error logs
+      description: |
+       If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````.
+
+      placeholder: |
+        ```
+        The error message you got, with the full traceback.
+        ```
+
+    validations:
+      required: true
+
+  
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: "A clear and concise description of what you would expect to happen."
+
+  - type: markdown
+    attributes:
+      value: >
+        Thanks for contributing 🎉!

Wav-Omni/.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,31 @@
+name: 🚀 Feature request
+description: Submit a proposal/request for a new Wav-Omni feature
+
+body:
+- type: textarea
+  id: feature-pitch
+  attributes:
+    label: 🚀 The feature, motivation and pitch
+    description: >
+      A clear and concise description of the feature proposal. Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., *"I'm working on X and would like Y to be possible"*. If this is related to another GitHub issue, please link here too.
+  validations:
+    required: true
+
+- type: textarea
+  id: alternatives
+  attributes:
+    label: Alternatives
+    description: >
+      A description of any alternative solutions or features you've considered, if any.
+
+- type: textarea
+  id: additional-context
+  attributes:
+    label: Additional context
+    description: >
+      Add any other context or screenshots about the feature request.
+
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

Wav-Omni/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,38 @@
+# What does this PR do?
+
+<!--
+Congratulations! You've made it this far! You're not quite done yet though.
+
+Please include a good title that fully reflects the extent of your awesome contribution.
+
+Then, please replace this with a description of the change and which issue is fixed (if applicable). Please also include relevant motivation and context. List any dependencies (if any) that are required for this change.
+
+-->
+
+<!-- Remove if not applicable -->
+
+Fixes # (issue)
+
+
+## Feature/Issue validation/testing
+
+Please describe the tests that you ran to verify your changes and relevant result summary. Provide instructions so it can be reproduced.
+Please also list any relevant details for your test configuration.
+
+- [ ] Test A
+Logs for Test A
+
+- [ ] Test B
+Logs for Test B
+
+
+## Before submitting
+- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
+- [ ] Did you read the [contributor guideline](https://github.com/facebookresearch/llama-recipes/blob/main/CONTRIBUTING.md#pull-requests),
+      Pull Request section?
+- [ ] Was this discussed/approved via a Github issue? Please add a link
+      to it if that's the case.
+- [ ] Did you make sure to update the documentation with your changes?  
+- [ ] Did you write any new necessary tests?
+
+Thanks for contributing 🎉!

Wav-Omni/.github/workflows/spellcheck.yml

@@ -0,0 +1,66 @@
+name: SpellCheck
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    name: Lint changed files
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # OR "2" -> To retrieve the preceding commit.
+
+      - name: Check links in all markdown files
+        uses: gaurav-nelson/github-action-markdown-link-check@1.0.13
+        with:
+          use-verbose-mode: 'yes'
+          config-file: "scripts/markdown_link_check_config.json"
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v29.0.4
+        with:
+
+          files: |
+            **/*.py
+
+  spellcheck:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get install aspell aspell-en
+          pip install pyspelling
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v29.0.4
+        with:
+          files: |
+            **/*.md
+
+      - name: Check spellings
+        run: |
+          sources=""
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            sources="${sources} -S $file"
+          done
+          if [ ! "$sources" ]; then
+            echo "No files to spellcheck"
+          else
+            pyspelling -c $GITHUB_WORKSPACE/scripts/spellcheck_conf/spellcheck.yaml --name Markdown $sources
+          fi
+
+      - name: In the case of misspellings
+        if: ${{ failure() }}
+        run: |
+          echo "Please fix the misspellings. If you are sure about some of them, "
+          echo "so append those to scripts/spellcheck_conf/wordlist.txt"

Wav-Omni/.gitignore

@@ -0,0 +1,25 @@
+.DS_Store
+__pycache__
+.ipynb_checkpoints
+.vscode
+debug.py
+debug.ipynb
+debug.sh
+.idea/*
+transformers
+wandb/
+log/
+*.log
+outputs/
+data/
+jobs/
+debug/
+audio/
+
+examples/s2s/scripts/debug
+examples/vsr_LRS3/scripts/decode_avhubert_vo_vicuna_7b_noself.sh
+examples/asr_librispeech/scripts/decode_hubert_xtralarge_linear_vicuna_7b_copy.sh
+examples/vsr_LRS3/scripts/decode_avhubert_vo_vicuna_7b_copy.sh
+scripts_all
+examples/hotwords_librispeech
+examples/asr_librispeech/scripts/decode_hubert_xtralarge_linear_vicuna_7b_debug.sh

Wav-Omni/Dockerfile

@@ -0,0 +1,41 @@
+FROM pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
+
+USER root
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+LABEL github_repo="https://github.com/ddlBoJack/Wav-Omni"
+
+RUN set -x \
+    && apt-get update \
+    && apt-get -y install wget curl man git less openssl libssl-dev unzip unar build-essential aria2 tmux vim ninja-build\
+    && apt-get install -y openssh-server sox libsox-fmt-all libsox-fmt-mp3 libsndfile1-dev ffmpeg \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+RUN pip install --no-cache-dir packaging editdistance gpustat wandb einops debugpy tqdm soundfile matplotlib scipy sentencepiece pandas \
+    && pip install --no-cache-dir torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu118
+
+WORKDIR /workspace
+
+RUN git clone https://github.com/huggingface/transformers.git \
+    && cd transformers \
+    && git checkout tags/v4.35.2 \
+    && pip install --no-cache-dir -e .
+
+RUN git clone https://github.com/huggingface/peft.git \
+    && cd peft \
+    && git checkout tags/v0.6.0 \
+    && pip install --no-cache-dir -e .
+
+RUN git clone https://github.com/pytorch/fairseq \
+    && cd fairseq \
+    && pip install --no-cache-dir --editable ./
+
+RUN git clone https://github.com/ddlBoJack/Wav-Omni.git \
+    && cd Wav-Omni \
+    && pip install --no-cache-dir -e .
+
+ENV SHELL=/bin/bash
+
+WORKDIR /workspace/Wav-Omni

Wav-Omni/README.md

@@ -0,0 +1,10 @@
+### 安装环境
+./Wav-Omni/environment.yml
+
+### 训练和推理
+bash ./Wav-Omni/train.sh
+bash ./Wav-Omni/infer.sh
+
+得huggingface上提前下载Qwen和cosyvoice的权重，替换配置文件中的路径。
+
+数据集中的方式可以按照demo给出的样例构建

Wav-Omni/ckpt/wavomni_chinese_3_2/.hydra/config.yaml

@@ -0,0 +1,68 @@
+dataset_config:
+  prompt: 'Conduct a spoken conversation with the user. '
+  dataset: speech_dataset_s2s
+  val_data_path: /home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+  train_data_path: /home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+  input_type: mel
+  mel_size: 128
+  inference_mode: true
+  manifest_format: jsonl
+  split_size: 0.002
+  load_from_cache_file: false
+  task_type: s2s
+  seed: 777
+  vocab_config:
+    code_layer: 3
+    total_audio_vocabsize: 4160
+    total_vocabsize: 156160
+  code_type: CosyVoice
+  num_latency_tokens: 0
+  do_layershift: false
+model_config:
+  llm_name: qwen2-0.5b
+  llm_path: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B
+  llm_dim: 896
+  encoder_name: whisper
+  encoder_projector_ds_rate: 5
+  encoder_path: large-v3
+  encoder_dim: 1280
+  encoder_projector: linear
+  codec_decoder_path: /home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT
+  codec_decode: true
+  vocab_config:
+    code_layer: 3
+    total_audio_vocabsize: 4160
+    total_vocabsize: 156160
+  code_type: CosyVoice
+  codec_decoder_type: CosyVoice
+  group_decode: true
+  group_decode_adapter_type: linear
+train_config:
+  model_name: s2s
+  freeze_encoder: true
+  freeze_llm: true
+  freeze_encoder_projector: true
+  freeze_group_decode_adapter: true
+  batching_strategy: custom
+  num_epochs: 1
+  val_batch_size: 1
+  num_workers_dataloader: 2
+  task_type: s2s
+decode_config:
+  text_repetition_penalty: 1.2
+  audio_repetition_penalty: 1.2
+  max_new_tokens: 3000
+  task_type: s2s
+  do_sample: false
+  top_p: 1.0
+  top_k: 0
+  temperature: 1.0
+  decode_text_only: false
+  do_layershift: false
+  num_latency_tokens: 0
+decode_log: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy
+ckpt_path: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+output_text_only: false
+inference_online: false
+speech_sample_rate: 22050
+audio_prompt_path: /home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav

Wav-Omni/ckpt/wavomni_chinese_3_2/.hydra/hydra.yaml

@@ -0,0 +1,216 @@
+hydra:
+  run:
+    dir: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2
+    - hydra.mode=RUN
+    task:
+    - ++model_config.llm_name=qwen2-0.5b
+    - ++model_config.llm_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B
+    - ++model_config.llm_dim=896
+    - ++model_config.encoder_name=whisper
+    - ++model_config.encoder_projector_ds_rate=5
+    - ++model_config.encoder_path=large-v3
+    - ++model_config.encoder_dim=1280
+    - ++model_config.encoder_projector=linear
+    - ++model_config.codec_decoder_path=/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT
+    - ++model_config.codec_decode=true
+    - ++model_config.vocab_config.code_layer=3
+    - ++model_config.vocab_config.total_audio_vocabsize=4160
+    - ++model_config.vocab_config.total_vocabsize=156160
+    - ++model_config.code_type=CosyVoice
+    - ++model_config.codec_decoder_type=CosyVoice
+    - ++model_config.group_decode=true
+    - ++model_config.group_decode_adapter_type=linear
+    - ++dataset_config.dataset=speech_dataset_s2s
+    - ++dataset_config.val_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+    - ++dataset_config.train_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+    - ++dataset_config.input_type=mel
+    - ++dataset_config.mel_size=128
+    - ++dataset_config.inference_mode=true
+    - ++dataset_config.manifest_format=jsonl
+    - ++dataset_config.split_size=0.002
+    - ++dataset_config.load_from_cache_file=false
+    - ++dataset_config.task_type=s2s
+    - ++dataset_config.seed=777
+    - ++dataset_config.vocab_config.code_layer=3
+    - ++dataset_config.vocab_config.total_audio_vocabsize=4160
+    - ++dataset_config.vocab_config.total_vocabsize=156160
+    - ++dataset_config.code_type=CosyVoice
+    - ++dataset_config.num_latency_tokens=0
+    - ++dataset_config.do_layershift=false
+    - ++train_config.model_name=s2s
+    - ++train_config.freeze_encoder=true
+    - ++train_config.freeze_llm=true
+    - ++train_config.freeze_encoder_projector=true
+    - ++train_config.freeze_group_decode_adapter=true
+    - ++train_config.batching_strategy=custom
+    - ++train_config.num_epochs=1
+    - ++train_config.val_batch_size=1
+    - ++train_config.num_workers_dataloader=2
+    - ++train_config.task_type=s2s
+    - ++decode_config.text_repetition_penalty=1.2
+    - ++decode_config.audio_repetition_penalty=1.2
+    - ++decode_config.max_new_tokens=3000
+    - ++decode_config.task_type=s2s
+    - ++decode_config.do_sample=false
+    - ++decode_config.top_p=1.0
+    - ++decode_config.top_k=0
+    - ++decode_config.temperature=1.0
+    - ++decode_config.decode_text_only=false
+    - ++decode_config.do_layershift=false
+    - ++decode_log=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy
+    - ++decode_config.num_latency_tokens=0
+    - ++ckpt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+    - ++output_text_only=false
+    - ++inference_online=false
+    - ++speech_sample_rate=22050
+    - ++audio_prompt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav
+  job:
+    name: inference_s2s
+    chdir: null
+    override_dirname: ++audio_prompt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav,++ckpt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt,++dataset_config.code_type=CosyVoice,++dataset_config.dataset=speech_dataset_s2s,++dataset_config.do_layershift=false,++dataset_config.inference_mode=true,++dataset_config.input_type=mel,++dataset_config.load_from_cache_file=false,++dataset_config.manifest_format=jsonl,++dataset_config.mel_size=128,++dataset_config.num_latency_tokens=0,++dataset_config.seed=777,++dataset_config.split_size=0.002,++dataset_config.task_type=s2s,++dataset_config.train_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl,++dataset_config.val_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl,++dataset_config.vocab_config.code_layer=3,++dataset_config.vocab_config.total_audio_vocabsize=4160,++dataset_config.vocab_config.total_vocabsize=156160,++decode_config.audio_repetition_penalty=1.2,++decode_config.decode_text_only=false,++decode_config.do_layershift=false,++decode_config.do_sample=false,++decode_config.max_new_tokens=3000,++decode_config.num_latency_tokens=0,++decode_config.task_type=s2s,++decode_config.temperature=1.0,++decode_config.text_repetition_penalty=1.2,++decode_config.top_k=0,++decode_config.top_p=1.0,++decode_log=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy,++inference_online=false,++model_config.code_type=CosyVoice,++model_config.codec_decode=true,++model_config.codec_decoder_path=/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT,++model_config.codec_decoder_type=CosyVoice,++model_config.encoder_dim=1280,++model_config.encoder_name=whisper,++model_config.encoder_path=large-v3,++model_config.encoder_projector=linear,++model_config.encoder_projector_ds_rate=5,++model_config.group_decode=true,++model_config.group_decode_adapter_type=linear,++model_config.llm_dim=896,++model_config.llm_name=qwen2-0.5b,++model_config.llm_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B,++model_config.vocab_config.code_layer=3,++model_config.vocab_config.total_audio_vocabsize=4160,++model_config.vocab_config.total_vocabsize=156160,++output_text_only=false,++speech_sample_rate=22050,++train_config.batching_strategy=custom,++train_config.freeze_encoder=true,++train_config.freeze_encoder_projector=true,++train_config.freeze_group_decode_adapter=true,++train_config.freeze_llm=true,++train_config.model_name=s2s,++train_config.num_epochs=1,++train_config.num_workers_dataloader=2,++train_config.task_type=s2s,++train_config.val_batch_size=1
+    id: ???
+    num: ???
+    config_name: prompt.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/xj_data/jishengpeng/huawei/Wav-Omni
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Wav-Omni/ckpt/wavomni_chinese_3_2/.hydra/overrides.yaml

@@ -0,0 +1,61 @@
+- ++model_config.llm_name=qwen2-0.5b
+- ++model_config.llm_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B
+- ++model_config.llm_dim=896
+- ++model_config.encoder_name=whisper
+- ++model_config.encoder_projector_ds_rate=5
+- ++model_config.encoder_path=large-v3
+- ++model_config.encoder_dim=1280
+- ++model_config.encoder_projector=linear
+- ++model_config.codec_decoder_path=/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT
+- ++model_config.codec_decode=true
+- ++model_config.vocab_config.code_layer=3
+- ++model_config.vocab_config.total_audio_vocabsize=4160
+- ++model_config.vocab_config.total_vocabsize=156160
+- ++model_config.code_type=CosyVoice
+- ++model_config.codec_decoder_type=CosyVoice
+- ++model_config.group_decode=true
+- ++model_config.group_decode_adapter_type=linear
+- ++dataset_config.dataset=speech_dataset_s2s
+- ++dataset_config.val_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+- ++dataset_config.train_data_path=/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl
+- ++dataset_config.input_type=mel
+- ++dataset_config.mel_size=128
+- ++dataset_config.inference_mode=true
+- ++dataset_config.manifest_format=jsonl
+- ++dataset_config.split_size=0.002
+- ++dataset_config.load_from_cache_file=false
+- ++dataset_config.task_type=s2s
+- ++dataset_config.seed=777
+- ++dataset_config.vocab_config.code_layer=3
+- ++dataset_config.vocab_config.total_audio_vocabsize=4160
+- ++dataset_config.vocab_config.total_vocabsize=156160
+- ++dataset_config.code_type=CosyVoice
+- ++dataset_config.num_latency_tokens=0
+- ++dataset_config.do_layershift=false
+- ++train_config.model_name=s2s
+- ++train_config.freeze_encoder=true
+- ++train_config.freeze_llm=true
+- ++train_config.freeze_encoder_projector=true
+- ++train_config.freeze_group_decode_adapter=true
+- ++train_config.batching_strategy=custom
+- ++train_config.num_epochs=1
+- ++train_config.val_batch_size=1
+- ++train_config.num_workers_dataloader=2
+- ++train_config.task_type=s2s
+- ++decode_config.text_repetition_penalty=1.2
+- ++decode_config.audio_repetition_penalty=1.2
+- ++decode_config.max_new_tokens=3000
+- ++decode_config.task_type=s2s
+- ++decode_config.do_sample=false
+- ++decode_config.top_p=1.0
+- ++decode_config.top_k=0
+- ++decode_config.temperature=1.0
+- ++decode_config.decode_text_only=false
+- ++decode_config.do_layershift=false
+- ++decode_log=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy
+- ++decode_config.num_latency_tokens=0
+- ++ckpt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+- ++output_text_only=false
+- ++inference_online=false
+- ++speech_sample_rate=22050
+- ++audio_prompt_path=/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav

Wav-Omni/ckpt/wavomni_chinese_3_2/inference_s2s.log

@@ -0,0 +1,175 @@
+[2025-03-05 13:27:43,011][root][INFO] - train_config: {'model_name': 's2s', 'enable_ddp': False, 'enable_deepspeed': False, 'enable_fsdp': False, 'low_cpu_fsdp': False, 'run_validation': True, 'batch_size_training': 4, 'batching_strategy': 'custom', 'context_length': 4096, 'gradient_accumulation_steps': 1, 'num_epochs': 1, 'num_workers_dataloader': 2, 'warmup_steps': 1000, 'total_steps': 100000, 'validation_interval': 1000, 'lr': 0.0001, 'weight_decay': 0.0, 'gamma': 0.85, 'seed': 42, 'use_fp16': False, 'mixed_precision': True, 'val_batch_size': 1, 'use_peft': False, 'peft_config': {'peft_method': 'lora', 'r': 8, 'lora_alpha': 32, 'target_modules': ['q_proj', 'v_proj'], 'bias': 'none', 'task_type': 'CAUSAL_LM', 'lora_dropout': 0.05, 'inference_mode': False}, 'output_dir': 'PATH/to/save/PEFT/model', 'freeze_layers': False, 'num_freeze_layers': 1, 'quantization': False, 'one_gpu': False, 'save_model': True, 'dist_checkpoint_root_folder': 'PATH/to/save/FSDP/model', 'dist_checkpoint_folder': 'fine-tuned', 'save_optimizer': False, 'use_fast_kernels': False, 'run_test_during_validation': False, 'run_test_during_validation_file': 'test.wav', 'run_test_during_validation_prompt': '<|S2S|>', 'freeze_llm': True, 'freeze_encoder': True, 'train_embed_only': False, 'train_audio_embed_only': False, 'train_embed': False, 'task_type': 's2s', 'freeze_encoder_projector': True, 'freeze_group_decode_adapter': True}
+[2025-03-05 13:27:43,011][root][INFO] - fsdp_config: {'mixed_precision': True, 'use_fp16': False, 'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'SHARDED_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'pure_bf16': False, 'optimizer': 'AdamW'}
+[2025-03-05 13:27:43,011][root][INFO] - model_config: {'file': 'examples/s2s/model/slam_model_s2s.py:model_factory', 'llm_name': 'qwen2-0.5b', 'llm_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B', 'llm_type': 'decoder_only', 'llm_dim': 896, 'encoder_name': 'whisper', 'encoder_ds_rate': 2, 'encoder_path': 'large-v3', 'encoder_dim': 1280, 'encoder_projector': 'linear', 'encoder_projector_ds_rate': 5, 'modal': 'audio', 'normalize': False, 'encoder_type': 'finetune', 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'codec_decode': True, 'codec_decoder_type': 'CosyVoice', 'codec_decoder_path': '/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT', 'tts_adapter': False, 'tts_adapter_config': {'add_qkv_bias': True, 'bias': False, 'gelu_approximate': None, 'head_size': 64, 'intermediate_size': 4864, 'lm_head_bias': False, 'mlp_class_name': 'GptNeoxMLP', 'n_layer': 6, 'n_head': 14, 'n_embd': 896, 'n_query_groups': 2, 'norm_class_name': 'RMSNorm', 'norm_eps': 1e-06, 'parallel_residual': False, 'rotary_percentage': 1.0, 'shared_attention_norm': False}, 'encoder_path_hf': None, 'code_type': 'CosyVoice', 'group_decode': True, 'group_decode_adapter_type': 'linear', 'whisper_decode': False}
+[2025-03-05 13:28:09,748][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 13:28:09,750][wavomni_llm.utils.train_utils][INFO] - --> whisper has 635.04896 Million params
+
+[2025-03-05 13:28:09,752][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 13:28:09,753][wavomni_llm.utils.train_utils][INFO] - --> whisper has 0.0 Million params
+
+[2025-03-05 13:28:33,421][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 13:28:33,423][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 494.032768 Million params
+
+[2025-03-05 13:28:33,424][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 13:28:33,425][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 0.0 Million params
+
+[2025-03-05 13:28:33,502][wavomni_llm.utils.train_utils][INFO] - --> Module linear
+[2025-03-05 13:28:33,502][wavomni_llm.utils.train_utils][INFO] - --> linear has 14.945152 Million params
+
+[2025-03-05 13:28:46,757][root][INFO] - input frame rate=50
+[2025-03-05 13:28:56,015][wetext-zh_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_tagger.fst
+[2025-03-05 13:28:56,015][wetext-zh_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_verbalizer.fst
+[2025-03-05 13:28:56,015][wetext-zh_normalizer][INFO] - skip building fst for zh_normalizer ...
+[2025-03-05 13:28:56,372][wetext-en_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_tagger.fst
+[2025-03-05 13:28:56,373][wetext-en_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_verbalizer.fst
+[2025-03-05 13:28:56,373][wetext-en_normalizer][INFO] - skip building fst for en_normalizer ...
+[2025-03-05 13:29:46,255][wavomni_llm.utils.train_utils][INFO] - --> Module CosyVoice Codec
+[2025-03-05 13:29:46,260][wavomni_llm.utils.train_utils][INFO] - --> CosyVoice Codec has 104.577327 Million params
+
+[2025-03-05 13:29:46,543][wavomni_llm.utils.train_utils][INFO] - --> Module GroupDecodeAdapter_linear
+[2025-03-05 13:29:46,543][wavomni_llm.utils.train_utils][INFO] - --> GroupDecodeAdapter_linear has 51.92928 Million params
+
+[2025-03-05 13:29:48,085][slam_model_s2s.py][INFO] - Resize llm embedding layer's vocab size to 156160
+
+[2025-03-05 13:29:48,085][slam_model_s2s.py][INFO] - loading other parts from: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+
+[2025-03-05 13:30:16,442][wavomni_llm.utils.train_utils][INFO] - --> Model s2s
+[2025-03-05 13:30:16,444][wavomni_llm.utils.train_utils][INFO] - --> s2s has 0.0 Million params
+
+[2025-03-05 13:30:17,360][root][INFO] - dataset_config: {'dataset': 'speech_dataset_s2s', 'file': 'examples/s2s/speech_dataset_s2s.py:get_speech_dataset', 'train_data_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'val_data_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'train_split': 'train', 'test_split': 'validation', 'prompt': 'Conduct a spoken conversation with the user. ', 'data_path': None, 'max_words': None, 'max_mel': None, 'fix_length_audio': -1, 'inference_mode': True, 'input_type': 'mel', 'mel_size': 128, 'normalize': False, 'seed': 777, 'manifest_format': 'jsonl', 'split_size': 0.002, 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'load_from_cache_file': False, 'task_type': 's2s', 'upsample_text_tokens': False, 'upsampling_factor': 1, 'upsample_method': 'repeat', 'code_type': 'CosyVoice', 'num_latency_tokens': 0, 'do_layershift': False}
+[2025-03-05 13:30:17,420][root][INFO] - --> Training Set Length = 4
+[2025-03-05 13:30:17,429][root][INFO] - decode_config: {'do_sample': False, 'max_new_tokens': 3000, 'min_length': 10, 'temperature': 1.0, 'top_k': 0, 'top_p': 1.0, 'num_beams': 1, 'num_return_sequences': 1, 'num_samples': 1, 'max_time': 0.0, 'text_repetition_penalty': 1.2, 'audio_repetition_penalty': 1.2, 'length_penalty': 1.0, 'early_stopping': False, 'no_repeat_ngram_size': 0, 'bad_words_ids': [], 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'task_type': 's2s', 'decode_text_only': False, 'streaming': False, 'stream_stride': 4, 'upsampling_factor': 1, 'input_text': False, 'do_layershift': False, 'num_latency_tokens': 0}
+[2025-03-05 13:30:17,429][root][INFO] - Decode Strategy: Greedy
+[2025-03-05 13:30:17,430][root][INFO] - Decode Text & Audio
+[2025-03-05 13:30:17,430][root][INFO] - Decode Code Type: CosyVoice
+[2025-03-05 13:30:17,430][root][INFO] - Decode Code Layer: 3
+[2025-03-05 13:30:17,430][root][INFO] - Tone for Audio Generation: prompt_1
+[2025-03-05 13:30:17,430][root][INFO] - ============== Start s2s Inference ==============
+[2025-03-05 13:30:38,187][root][INFO] - LLM Inference Time: 19.46s
+[2025-03-05 13:30:38,188][root][INFO] - Question: 回答该题：柠檬的科学名称是什么？
+[2025-03-05 13:30:38,188][root][INFO] - Generated Text: 柠檬的科学名称是“柑橘科柠檬”，也被称为“柑橘属柠檬”。柑橘科柠檬是一种常见的水果，其果实通常呈圆锥形或椭圆形，并且含有大量的汁液。它富含维生素C、钾和纤维素等营养成分，对身体健康有很好的保健作用。柑橘科柠檬也被广泛用于烹饪和饮料制作中，被人们认为具有美容养颜功效。
+[2025-03-05 13:30:44,356][root][INFO] - Generated Audio: prompt_1/1.wav, audio length: 25.80s, generation time: 25.60s, RTF: 0.99
+[2025-03-05 13:30:44,356][root][INFO] - LLM RTF: 0.75
+[2025-03-05 13:57:49,762][root][INFO] - train_config: {'model_name': 's2s', 'enable_ddp': False, 'enable_deepspeed': False, 'enable_fsdp': False, 'low_cpu_fsdp': False, 'run_validation': True, 'batch_size_training': 4, 'batching_strategy': 'custom', 'context_length': 4096, 'gradient_accumulation_steps': 1, 'num_epochs': 1, 'num_workers_dataloader': 2, 'warmup_steps': 1000, 'total_steps': 100000, 'validation_interval': 1000, 'lr': 0.0001, 'weight_decay': 0.0, 'gamma': 0.85, 'seed': 42, 'use_fp16': False, 'mixed_precision': True, 'val_batch_size': 1, 'use_peft': False, 'peft_config': {'peft_method': 'lora', 'r': 8, 'lora_alpha': 32, 'target_modules': ['q_proj', 'v_proj'], 'bias': 'none', 'task_type': 'CAUSAL_LM', 'lora_dropout': 0.05, 'inference_mode': False}, 'output_dir': 'PATH/to/save/PEFT/model', 'freeze_layers': False, 'num_freeze_layers': 1, 'quantization': False, 'one_gpu': False, 'save_model': True, 'dist_checkpoint_root_folder': 'PATH/to/save/FSDP/model', 'dist_checkpoint_folder': 'fine-tuned', 'save_optimizer': False, 'use_fast_kernels': False, 'run_test_during_validation': False, 'run_test_during_validation_file': 'test.wav', 'run_test_during_validation_prompt': '<|S2S|>', 'freeze_llm': True, 'freeze_encoder': True, 'train_embed_only': False, 'train_audio_embed_only': False, 'train_embed': False, 'task_type': 's2s', 'freeze_encoder_projector': True, 'freeze_group_decode_adapter': True}
+[2025-03-05 13:57:49,762][root][INFO] - fsdp_config: {'mixed_precision': True, 'use_fp16': False, 'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'SHARDED_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'pure_bf16': False, 'optimizer': 'AdamW'}
+[2025-03-05 13:57:49,762][root][INFO] - model_config: {'file': 'examples/s2s/model/slam_model_s2s.py:model_factory', 'llm_name': 'qwen2-0.5b', 'llm_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B', 'llm_type': 'decoder_only', 'llm_dim': 896, 'encoder_name': 'whisper', 'encoder_ds_rate': 2, 'encoder_path': 'large-v3', 'encoder_dim': 1280, 'encoder_projector': 'linear', 'encoder_projector_ds_rate': 5, 'modal': 'audio', 'normalize': False, 'encoder_type': 'finetune', 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'codec_decode': True, 'codec_decoder_type': 'CosyVoice', 'codec_decoder_path': '/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT', 'tts_adapter': False, 'tts_adapter_config': {'add_qkv_bias': True, 'bias': False, 'gelu_approximate': None, 'head_size': 64, 'intermediate_size': 4864, 'lm_head_bias': False, 'mlp_class_name': 'GptNeoxMLP', 'n_layer': 6, 'n_head': 14, 'n_embd': 896, 'n_query_groups': 2, 'norm_class_name': 'RMSNorm', 'norm_eps': 1e-06, 'parallel_residual': False, 'rotary_percentage': 1.0, 'shared_attention_norm': False}, 'encoder_path_hf': None, 'code_type': 'CosyVoice', 'group_decode': True, 'group_decode_adapter_type': 'linear', 'whisper_decode': False}
+[2025-03-05 13:58:15,152][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 13:58:15,154][wavomni_llm.utils.train_utils][INFO] - --> whisper has 635.04896 Million params
+
+[2025-03-05 13:58:15,157][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 13:58:15,158][wavomni_llm.utils.train_utils][INFO] - --> whisper has 0.0 Million params
+
+[2025-03-05 13:58:17,223][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 13:58:17,224][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 494.032768 Million params
+
+[2025-03-05 13:58:17,226][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 13:58:17,227][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 0.0 Million params
+
+[2025-03-05 13:58:17,322][wavomni_llm.utils.train_utils][INFO] - --> Module linear
+[2025-03-05 13:58:17,322][wavomni_llm.utils.train_utils][INFO] - --> linear has 14.945152 Million params
+
+[2025-03-05 13:58:27,777][root][INFO] - input frame rate=50
+[2025-03-05 13:58:30,984][wetext-zh_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_tagger.fst
+[2025-03-05 13:58:30,984][wetext-zh_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_verbalizer.fst
+[2025-03-05 13:58:30,984][wetext-zh_normalizer][INFO] - skip building fst for zh_normalizer ...
+[2025-03-05 13:58:31,355][wetext-en_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_tagger.fst
+[2025-03-05 13:58:31,355][wetext-en_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_verbalizer.fst
+[2025-03-05 13:58:31,355][wetext-en_normalizer][INFO] - skip building fst for en_normalizer ...
+[2025-03-05 13:58:35,642][wavomni_llm.utils.train_utils][INFO] - --> Module CosyVoice Codec
+[2025-03-05 13:58:35,647][wavomni_llm.utils.train_utils][INFO] - --> CosyVoice Codec has 104.577327 Million params
+
+[2025-03-05 13:58:35,932][wavomni_llm.utils.train_utils][INFO] - --> Module GroupDecodeAdapter_linear
+[2025-03-05 13:58:35,932][wavomni_llm.utils.train_utils][INFO] - --> GroupDecodeAdapter_linear has 51.92928 Million params
+
+[2025-03-05 13:58:37,463][slam_model_s2s.py][INFO] - Resize llm embedding layer's vocab size to 156160
+
+[2025-03-05 13:58:37,463][slam_model_s2s.py][INFO] - loading other parts from: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+
+[2025-03-05 13:58:38,622][wavomni_llm.utils.train_utils][INFO] - --> Model s2s
+[2025-03-05 13:58:38,625][wavomni_llm.utils.train_utils][INFO] - --> s2s has 0.0 Million params
+
+[2025-03-05 13:58:39,524][root][INFO] - dataset_config: {'dataset': 'speech_dataset_s2s', 'file': 'examples/s2s/speech_dataset_s2s.py:get_speech_dataset', 'train_data_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'val_data_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'train_split': 'train', 'test_split': 'validation', 'prompt': 'Conduct a spoken conversation with the user. ', 'data_path': None, 'max_words': None, 'max_mel': None, 'fix_length_audio': -1, 'inference_mode': True, 'input_type': 'mel', 'mel_size': 128, 'normalize': False, 'seed': 777, 'manifest_format': 'jsonl', 'split_size': 0.002, 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'load_from_cache_file': False, 'task_type': 's2s', 'upsample_text_tokens': False, 'upsampling_factor': 1, 'upsample_method': 'repeat', 'code_type': 'CosyVoice', 'num_latency_tokens': 0, 'do_layershift': False}
+[2025-03-05 13:58:39,527][root][INFO] - --> Training Set Length = 4
+[2025-03-05 13:58:39,530][root][INFO] - decode_config: {'do_sample': False, 'max_new_tokens': 3000, 'min_length': 10, 'temperature': 1.0, 'top_k': 0, 'top_p': 1.0, 'num_beams': 1, 'num_return_sequences': 1, 'num_samples': 1, 'max_time': 0.0, 'text_repetition_penalty': 1.2, 'audio_repetition_penalty': 1.2, 'length_penalty': 1.0, 'early_stopping': False, 'no_repeat_ngram_size': 0, 'bad_words_ids': [], 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'task_type': 's2s', 'decode_text_only': False, 'streaming': False, 'stream_stride': 4, 'upsampling_factor': 1, 'input_text': False, 'do_layershift': False, 'num_latency_tokens': 0}
+[2025-03-05 13:58:39,530][root][INFO] - Decode Strategy: Greedy
+[2025-03-05 13:58:39,530][root][INFO] - Decode Text & Audio
+[2025-03-05 13:58:39,530][root][INFO] - Decode Code Type: CosyVoice
+[2025-03-05 13:58:39,530][root][INFO] - Decode Code Layer: 3
+[2025-03-05 13:58:39,530][root][INFO] - Tone for Audio Generation: prompt_1
+[2025-03-05 13:58:39,530][root][INFO] - ============== Start s2s Inference ==============
+[2025-03-05 14:00:42,800][root][INFO] - train_config: {'model_name': 's2s', 'enable_ddp': False, 'enable_deepspeed': False, 'enable_fsdp': False, 'low_cpu_fsdp': False, 'run_validation': True, 'batch_size_training': 4, 'batching_strategy': 'custom', 'context_length': 4096, 'gradient_accumulation_steps': 1, 'num_epochs': 1, 'num_workers_dataloader': 2, 'warmup_steps': 1000, 'total_steps': 100000, 'validation_interval': 1000, 'lr': 0.0001, 'weight_decay': 0.0, 'gamma': 0.85, 'seed': 42, 'use_fp16': False, 'mixed_precision': True, 'val_batch_size': 1, 'use_peft': False, 'peft_config': {'peft_method': 'lora', 'r': 8, 'lora_alpha': 32, 'target_modules': ['q_proj', 'v_proj'], 'bias': 'none', 'task_type': 'CAUSAL_LM', 'lora_dropout': 0.05, 'inference_mode': False}, 'output_dir': 'PATH/to/save/PEFT/model', 'freeze_layers': False, 'num_freeze_layers': 1, 'quantization': False, 'one_gpu': False, 'save_model': True, 'dist_checkpoint_root_folder': 'PATH/to/save/FSDP/model', 'dist_checkpoint_folder': 'fine-tuned', 'save_optimizer': False, 'use_fast_kernels': False, 'run_test_during_validation': False, 'run_test_during_validation_file': 'test.wav', 'run_test_during_validation_prompt': '<|S2S|>', 'freeze_llm': True, 'freeze_encoder': True, 'train_embed_only': False, 'train_audio_embed_only': False, 'train_embed': False, 'task_type': 's2s', 'freeze_encoder_projector': True, 'freeze_group_decode_adapter': True}
+[2025-03-05 14:00:42,800][root][INFO] - fsdp_config: {'mixed_precision': True, 'use_fp16': False, 'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'SHARDED_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'pure_bf16': False, 'optimizer': 'AdamW'}
+[2025-03-05 14:00:42,800][root][INFO] - model_config: {'file': 'examples/s2s/model/slam_model_s2s.py:model_factory', 'llm_name': 'qwen2-0.5b', 'llm_path': '/home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/Qwen2-0.5B', 'llm_type': 'decoder_only', 'llm_dim': 896, 'encoder_name': 'whisper', 'encoder_ds_rate': 2, 'encoder_path': 'large-v3', 'encoder_dim': 1280, 'encoder_projector': 'linear', 'encoder_projector_ds_rate': 5, 'modal': 'audio', 'normalize': False, 'encoder_type': 'finetune', 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'codec_decode': True, 'codec_decoder_type': 'CosyVoice', 'codec_decoder_path': '/home/xj_data/jishengpeng/CosyVoice/pretrained_models/CosyVoice-300M-SFT', 'tts_adapter': False, 'tts_adapter_config': {'add_qkv_bias': True, 'bias': False, 'gelu_approximate': None, 'head_size': 64, 'intermediate_size': 4864, 'lm_head_bias': False, 'mlp_class_name': 'GptNeoxMLP', 'n_layer': 6, 'n_head': 14, 'n_embd': 896, 'n_query_groups': 2, 'norm_class_name': 'RMSNorm', 'norm_eps': 1e-06, 'parallel_residual': False, 'rotary_percentage': 1.0, 'shared_attention_norm': False}, 'encoder_path_hf': None, 'code_type': 'CosyVoice', 'group_decode': True, 'group_decode_adapter_type': 'linear', 'whisper_decode': False}
+[2025-03-05 14:01:06,313][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 14:01:06,315][wavomni_llm.utils.train_utils][INFO] - --> whisper has 635.04896 Million params
+
+[2025-03-05 14:01:06,316][wavomni_llm.utils.train_utils][INFO] - --> Module whisper
+[2025-03-05 14:01:06,317][wavomni_llm.utils.train_utils][INFO] - --> whisper has 0.0 Million params
+
+[2025-03-05 14:01:08,196][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 14:01:08,197][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 494.032768 Million params
+
+[2025-03-05 14:01:08,198][wavomni_llm.utils.train_utils][INFO] - --> Module qwen2-0.5b
+[2025-03-05 14:01:08,199][wavomni_llm.utils.train_utils][INFO] - --> qwen2-0.5b has 0.0 Million params
+
+[2025-03-05 14:01:08,274][wavomni_llm.utils.train_utils][INFO] - --> Module linear
+[2025-03-05 14:01:08,274][wavomni_llm.utils.train_utils][INFO] - --> linear has 14.945152 Million params
+
+[2025-03-05 14:01:17,364][root][INFO] - input frame rate=50
+[2025-03-05 14:01:20,258][wetext-zh_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_tagger.fst
+[2025-03-05 14:01:20,258][wetext-zh_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/zh_tn_verbalizer.fst
+[2025-03-05 14:01:20,258][wetext-zh_normalizer][INFO] - skip building fst for zh_normalizer ...
+[2025-03-05 14:01:20,643][wetext-en_normalizer][INFO] - found existing fst: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_tagger.fst
+[2025-03-05 14:01:20,643][wetext-en_normalizer][INFO] -                     /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni/lib/python3.9/site-packages/tn/en_tn_verbalizer.fst
+[2025-03-05 14:01:20,643][wetext-en_normalizer][INFO] - skip building fst for en_normalizer ...
+[2025-03-05 14:01:24,558][wavomni_llm.utils.train_utils][INFO] - --> Module CosyVoice Codec
+[2025-03-05 14:01:24,563][wavomni_llm.utils.train_utils][INFO] - --> CosyVoice Codec has 104.577327 Million params
+
+[2025-03-05 14:01:24,850][wavomni_llm.utils.train_utils][INFO] - --> Module GroupDecodeAdapter_linear
+[2025-03-05 14:01:24,850][wavomni_llm.utils.train_utils][INFO] - --> GroupDecodeAdapter_linear has 51.92928 Million params
+
+[2025-03-05 14:01:26,392][slam_model_s2s.py][INFO] - Resize llm embedding layer's vocab size to 156160
+
+[2025-03-05 14:01:26,393][slam_model_s2s.py][INFO] - loading other parts from: /home/xj_data/jishengpeng/huawei/Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt
+
+[2025-03-05 14:01:27,554][wavomni_llm.utils.train_utils][INFO] - --> Model s2s
+[2025-03-05 14:01:27,556][wavomni_llm.utils.train_utils][INFO] - --> s2s has 0.0 Million params
+
+[2025-03-05 14:01:28,452][root][INFO] - dataset_config: {'dataset': 'speech_dataset_s2s', 'file': 'examples/s2s/speech_dataset_s2s.py:get_speech_dataset', 'train_data_path': '/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'val_data_path': '/home/xj_data/jishengpeng/Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl', 'train_split': 'train', 'test_split': 'validation', 'prompt': 'Conduct a spoken conversation with the user. ', 'data_path': None, 'max_words': None, 'max_mel': None, 'fix_length_audio': -1, 'inference_mode': True, 'input_type': 'mel', 'mel_size': 128, 'normalize': False, 'seed': 777, 'manifest_format': 'jsonl', 'split_size': 0.002, 'vocab_config': {'text_vocabsize': 151936, 'text_specialtokens': 64, 'audio_vocabsize': 4096, 'audio_specialtokens': 64, 'code_layer': 3, 'padded_text_vocabsize': 152000, 'padded_audio_vocabsize': 4160, 'total_audio_vocabsize': 4160, 'total_vocabsize': 156160, 'eot': 151936, 'pad_t': 151937, 'input_t': 151938, 'answer_t': 151939, 'asr': 151940, 'eoa': 4096, 'pad_a': 4097, 'input_a': 4098, 'answer_a': 4099, 'split': 4100}, 'load_from_cache_file': False, 'task_type': 's2s', 'upsample_text_tokens': False, 'upsampling_factor': 1, 'upsample_method': 'repeat', 'code_type': 'CosyVoice', 'num_latency_tokens': 0, 'do_layershift': False}
+[2025-03-05 14:01:28,499][root][INFO] - --> Training Set Length = 4
+[2025-03-05 14:01:28,502][root][INFO] - decode_config: {'do_sample': False, 'max_new_tokens': 3000, 'min_length': 10, 'temperature': 1.0, 'top_k': 0, 'top_p': 1.0, 'num_beams': 1, 'num_return_sequences': 1, 'num_samples': 1, 'max_time': 0.0, 'text_repetition_penalty': 1.2, 'audio_repetition_penalty': 1.2, 'length_penalty': 1.0, 'early_stopping': False, 'no_repeat_ngram_size': 0, 'bad_words_ids': [], 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'task_type': 's2s', 'decode_text_only': False, 'streaming': False, 'stream_stride': 4, 'upsampling_factor': 1, 'input_text': False, 'do_layershift': False, 'num_latency_tokens': 0}
+[2025-03-05 14:01:28,502][root][INFO] - Decode Strategy: Greedy
+[2025-03-05 14:01:28,503][root][INFO] - Decode Text & Audio
+[2025-03-05 14:01:28,503][root][INFO] - Decode Code Type: CosyVoice
+[2025-03-05 14:01:28,503][root][INFO] - Decode Code Layer: 3
+[2025-03-05 14:01:28,503][root][INFO] - Tone for Audio Generation: prompt_1
+[2025-03-05 14:01:28,503][root][INFO] - ============== Start s2s Inference ==============
+[2025-03-05 14:01:49,477][root][INFO] - LLM Inference Time: 19.84s
+[2025-03-05 14:01:49,478][root][INFO] - Question: 回答该题：柠檬的科学名称是什么？
+[2025-03-05 14:01:49,478][root][INFO] - Generated Text: 柠檬的科学名称是“柑橘科柠檬”，也被称为“柑橘属柠檬”。柑橘科柠檬是一种常见的水果，其果实通常呈圆锥形或椭圆形，并且含有大量的汁液。它富含维生素C、钾和纤维素等营养成分，对身体健康有很好的保健作用。柑橘科柠檬也被广泛用于烹饪和饮料制作中，被人们认为具有美容养颜功效。
+[2025-03-05 14:01:55,437][root][INFO] - Generated Audio: prompt_1/1.wav, audio length: 25.80s, generation time: 25.76s, RTF: 1.00
+[2025-03-05 14:01:55,437][root][INFO] - LLM RTF: 0.77
+[2025-03-05 14:02:07,906][root][INFO] - LLM Inference Time: 12.47s
+[2025-03-05 14:02:07,906][root][INFO] - Question: 请提出五种减少塑料垃圾的方法。
+[2025-03-05 14:02:07,906][root][INFO] - Generated Text: 1. 使用可重复使用的购物袋和水瓶
+2. 减少使用一次性塑料制品，如塑料餐具、瓶子等
+3. 避免购买包装过多的食品，选择天然食材制作的食物
+4. 选择购买环保型产品，如有机纸袋、玻璃杯等
+5. 在家中安装玻璃窗或窗户，减少室内空气流通。
+[2025-03-05 14:02:16,500][root][INFO] - Generated Audio: prompt_1/2.wav, audio length: 23.09s, generation time: 21.04s, RTF: 0.91
+[2025-03-05 14:02:16,500][root][INFO] - LLM RTF: 0.54
+[2025-03-05 14:02:30,030][root][INFO] - LLM Inference Time: 13.53s
+[2025-03-05 14:02:30,030][root][INFO] - Question: 小李的薪水是每月8000元，他每个月需要交400元的养老保险和200元的医疗保险，请问他每个月实际能拿到的薪水是多少？
+[2025-03-05 14:02:30,030][root][INFO] - Generated Text: 小李每月实际能拿到的薪水是8000元+400元+200元=16000元。
+解题过程：
+1. 小李每个月需要交的养老保险和医疗保健的金额之和为 16000元
+2. 计算小李每月实际能拿到的薪水，即 16000元 - 15000元 = 15000元
+因此，小李每月实际能拿到的薪水是15000元。
+[2025-03-05 14:02:38,316][root][INFO] - Generated Audio: prompt_1/3.wav, audio length: 25.14s, generation time: 21.79s, RTF: 0.87
+[2025-03-05 14:02:38,316][root][INFO] - LLM RTF: 0.54
+[2025-03-05 14:02:48,677][root][INFO] - LLM Inference Time: 10.36s
+[2025-03-05 14:02:48,677][root][INFO] - Question: 你好，我现在是浙江大学计算机二年级的硕士生，攻读语音方向，你能给我推荐一些学习的建议吗？
+[2025-03-05 14:02:48,677][root][INFO] - Generated Text: 当然可以。首先，建议你多读一些经典的数学书籍，例如《数学原理》、《代数与几何》等。其次，要注重练习和思考，可以通过实践项目或者项目来巩固知识。最后，不要忘记在学习过程中保持耐心和积极性，因为只有通过不断的努力才能取得好的成果。
+[2025-03-05 14:02:57,149][root][INFO] - Generated Audio: prompt_1/4.wav, audio length: 19.41s, generation time: 18.80s, RTF: 0.97
+[2025-03-05 14:02:57,150][root][INFO] - LLM RTF: 0.53
+[2025-03-05 14:02:57,379][root][INFO] - ============== Inference Finished ==============

Wav-Omni/ckpt/wavomni_chinese_3_2/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd393406fe8e519802f73520504470f109a67519a3ab5d5442274793cace3bc
+size 2258846794

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/gt_text

@@ -0,0 +1,4 @@
+1	柠檬的科学名称为Citrus limon，属于芸香科柑橘属的常绿小乔木或灌木植物。它起源于亚洲南部，并在欧洲和地中海地区以及美洲热带地区广泛栽培。柠檬树可以在强烈的阳光下生长，所以它经常被栽种在阳光充足的地方，如地中海气候区。柠檬的果实是圆锥形的，皮表面略带纵行皱纹，果肉酸爽，用于制作饮料和调味料等。柠檬也是一种富含维生素C和抗氧化剂的水果，对保持健康有很大的益处。
+2	1. 减少使用塑料制品，转而使用可再生或可分解材料制品。2. 训练良好的垃圾分类习惯，将可回收和可分解的塑料垃圾分别分类处理。3. 限制超市、商场，以及快餐店等商家使用一次性塑料袋、餐具和纸杯等，鼓励折叠包与可重复使用的餐具。4. 呼吁政府出台相关的环境保护法律法规，限制塑料垃圾排放，处罚违规行为。5. 开展垃圾分类教育活动，吸引公众关注塑料垃圾减量，培养环保意识，减少不必要的消费行为
+3	小李每个月的薪水是8000元，需要交400元的养老保险和200元的医疗保险。因此，他实际能拿到的薪水是多少呢？我们可以用一个简单的计算公式来计算：实际薪水 = 总薪水 - 养老保险 - 医疗保险。将小李每月的总薪水8000元代入公式，养老保险400元和医疗保险200元分别代入得：实际薪水 = 8000 - 400 - 200 = 7400元，所以小李每个月实际能拿到的薪水是7400元。
+4	没有标准回答，请随意发挥。

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6a88562a43f4076affb94dada09d329021ecd7ad041bb2b1b67c22fc8fb3345
+size 1137708

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94d7a467901758c23a0fc998b44542b7f46f521ce2cb05fe36a1cdc485c9f04
+size 1018412

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/3.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e7f3a3d1c9cf1da6280012358379bcb0af32d7d2f4556fcd19de66b114a3676
+size 1108524

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_audio/prompt_1/4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:101fc7477936e0d7ab017d8b2ac78837a0f539a814bdd7df40004e6d3d974eb8
+size 856108

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/pred_text

@@ -0,0 +1,12 @@
+1	柠檬的科学名称是“柑橘科柠檬”，也被称为“柑橘属柠檬”。柑橘科柠檬是一种常见的水果，其果实通常呈圆锥形或椭圆形，并且含有大量的汁液。它富含维生素C、钾和纤维素等营养成分，对身体健康有很好的保健作用。柑橘科柠檬也被广泛用于烹饪和饮料制作中，被人们认为具有美容养颜功效。
+2	1. 使用可重复使用的购物袋和水瓶
+2. 减少使用一次性塑料制品，如塑料餐具、瓶子等
+3. 避免购买包装过多的食品，选择天然食材制作的食物
+4. 选择购买环保型产品，如有机纸袋、玻璃杯等
+5. 在家中安装玻璃窗或窗户，减少室内空气流通。
+3	小李每月实际能拿到的薪水是8000元+400元+200元=16000元。
+解题过程：
+1. 小李每个月需要交的养老保险和医疗保健的金额之和为 16000元
+2. 计算小李每月实际能拿到的薪水，即 16000元 - 15000元 = 15000元
+因此，小李每月实际能拿到的薪水是15000元。
+4	当然可以。首先，建议你多读一些经典的数学书籍，例如《数学原理》、《代数与几何》等。其次，要注重练习和思考，可以通过实践项目或者项目来巩固知识。最后，不要忘记在学习过程中保持耐心和积极性，因为只有通过不断的努力才能取得好的成果。

Wav-Omni/ckpt/wavomni_chinese_3_2/s2s_decode_jsp_trp1.2_arp1.2_seed777_greedy/question_text

@@ -0,0 +1,4 @@
+1	回答该题：柠檬的科学名称是什么？
+2	请提出五种减少塑料垃圾的方法。
+3	小李的薪水是每月8000元，他每个月需要交400元的养老保险和200元的医疗保险，请问他每个月实际能拿到的薪水是多少？
+4	你好，我现在是浙江大学计算机二年级的硕士生，攻读语音方向，你能给我推荐一些学习的建议吗？

Wav-Omni/environment.yml

@@ -0,0 +1,243 @@
+name: wavomni
+channels:
+  - defaults
+  - https://repo.anaconda.com/pkgs/main
+  - https://repo.anaconda.com/pkgs/r
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - ca-certificates=2024.12.31=h06a4308_0
+  - ld_impl_linux-64=2.40=h12ee557_0
+  - libffi=3.4.4=h6a678d5_1
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - ncurses=6.4=h6a678d5_0
+  - openssl=3.0.15=h5eee18b_0
+  - pip=25.0=py39h06a4308_0
+  - python=3.9.21=he870216_1
+  - readline=8.2=h5eee18b_0
+  - setuptools=75.8.0=py39h06a4308_0
+  - sqlite=3.45.3=h5eee18b_0
+  - tk=8.6.14=h39e8969_0
+  - wheel=0.45.1=py39h06a4308_0
+  - xz=5.4.6=h5eee18b_1
+  - zlib=1.2.13=h5eee18b_1
+  - pip:
+      - accelerate==1.3.0
+      - addict==2.4.0
+      - aiofiles==23.2.1
+      - aiohappyeyeballs==2.4.4
+      - aiohttp==3.11.12
+      - aiosignal==1.3.2
+      - aliyun-python-sdk-core==2.16.0
+      - aliyun-python-sdk-kms==2.16.5
+      - annotated-types==0.7.0
+      - antlr4-python3-runtime==4.9.3
+      - anyio==4.8.0
+      - appdirs==1.4.4
+      - asttokens==3.0.0
+      - async-timeout==5.0.1
+      - attrs==25.1.0
+      - audioread==3.0.1
+      - beautifulsoup4==4.13.3
+      - bitsandbytes==0.45.2
+      - black==25.1.0
+      - brotli==1.1.0
+      - certifi==2025.1.31
+      - cffi==1.17.1
+      - charset-normalizer==3.4.1
+      - click==8.1.8
+      - cmake==3.25.0
+      - coloredlogs==15.0.1
+      - conformer==0.3.2
+      - contourpy==1.3.0
+      - crcmod==1.7
+      - cryptography==44.0.0
+      - cycler==0.12.1
+      - cython==3.0.11
+      - datasets==2.18.0
+      - decorator==5.1.1
+      - deepspeed==0.14.2
+      - diffusers==0.27.2
+      - dill==0.3.8
+      - docker-pycreds==0.4.0
+      - einops==0.8.0
+      - eval-type-backport==0.2.2
+      - exceptiongroup==1.2.2
+      - executing==2.2.0
+      - fastapi==0.115.8
+      - ffmpy==0.5.0
+      - filelock==3.17.0
+      - fire==0.7.0
+      - flatbuffers==25.1.24
+      - fonttools==4.55.8
+      - frozenlist==1.5.0
+      - fsspec==2024.2.0
+      - gast==0.6.0
+      - gdown==5.2.0
+      - gitdb==4.0.12
+      - gitpython==3.1.44
+      - gradio==4.44.1
+      - gradio-client==1.3.0
+      - grpcio==1.57.0
+      - grpcio-tools==1.57.0
+      - h11==0.14.0
+      - hjson==3.1.0
+      - httpcore==1.0.7
+      - httpx==0.28.1
+      - huggingface-hub==0.25.2
+      - humanfriendly==10.0
+      - hydra-core==1.3.2
+      - hyperpyyaml==1.2.2
+      - idna==3.10
+      - importlib-metadata==8.6.1
+      - importlib-resources==6.5.2
+      - inflate64==1.0.1
+      - inflect==7.3.1
+      - ipython==8.18.1
+      - jedi==0.19.2
+      - jinja2==3.1.5
+      - jmespath==0.10.0
+      - joblib==1.4.2
+      - kiwisolver==1.4.7
+      - lazy-loader==0.4
+      - librosa==0.10.2.post1
+      - lightning==2.2.4
+      - lightning-utilities==0.12.0
+      - lit==15.0.7
+      - llvmlite==0.43.0
+      - loralib==0.1.2
+      - markdown-it-py==3.0.0
+      - markupsafe==2.1.5
+      - matplotlib==3.7.5
+      - matplotlib-inline==0.1.7
+      - mdurl==0.1.2
+      - modelscope==1.15.0
+      - more-itertools==10.6.0
+      - mpmath==1.3.0
+      - msgpack==1.1.0
+      - multidict==6.1.0
+      - multiprocess==0.70.16
+      - multivolumefile==0.2.3
+      - mypy-extensions==1.0.0
+      - networkx==3.2.1
+      - ninja==1.11.1.3
+      - numba==0.60.0
+      - numpy==1.26.4
+      - nvidia-cublas-cu12==12.1.3.1
+      - nvidia-cuda-cupti-cu12==12.1.105
+      - nvidia-cuda-nvrtc-cu12==12.1.105
+      - nvidia-cuda-runtime-cu12==12.1.105
+      - nvidia-cudnn-cu12==8.9.2.26
+      - nvidia-cufft-cu12==11.0.2.54
+      - nvidia-curand-cu12==10.3.2.106
+      - nvidia-cusolver-cu12==11.4.5.107
+      - nvidia-cusparse-cu12==12.1.0.106
+      - nvidia-cusparselt-cu12==0.6.2
+      - nvidia-ml-py==12.570.86
+      - nvidia-nccl-cu12==2.20.5
+      - nvidia-nvjitlink-cu12==12.4.127
+      - nvidia-nvtx-cu12==12.1.105
+      - omegaconf==2.3.0
+      - onnxruntime-gpu==1.18.0
+      - openai-whisper==20231117
+      - optimum==1.24.0
+      - orjson==3.10.15
+      - oss2==2.19.1
+      - packaging==24.2
+      - pandas==2.2.3
+      - parso==0.8.4
+      - pathspec==0.12.1
+      - peft==0.6.0
+      - pexpect==4.9.0
+      - pillow==10.4.0
+      - platformdirs==4.3.6
+      - pooch==1.8.2
+      - prompt-toolkit==3.0.50
+      - propcache==0.2.1
+      - protobuf==4.25.6
+      - psutil==6.1.1
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.3
+      - py-cpuinfo==9.0.0
+      - py7zr==0.22.0
+      - pyarrow==19.0.0
+      - pyarrow-hotfix==0.6
+      - pybcj==1.0.3
+      - pycparser==2.22
+      - pycryptodome==3.21.0
+      - pycryptodomex==3.21.0
+      - pydantic==2.10.6
+      - pydantic-core==2.27.2
+      - pydub==0.25.1
+      - pygments==2.19.1
+      - pynini==2.1.5
+      - pynvml==12.0.0
+      - pyparsing==3.2.1
+      - pyppmd==1.1.1
+      - pysocks==1.7.1
+      - python-dateutil==2.9.0.post0
+      - python-multipart==0.0.20
+      - pytorch-lightning==2.5.0.post0
+      - pytz==2025.1
+      - pyyaml==6.0.2
+      - pyzstd==0.16.2
+      - regex==2024.11.6
+      - requests==2.32.3
+      - rich==13.9.4
+      - ruamel-yaml==0.18.10
+      - ruamel-yaml-clib==0.2.12
+      - ruff==0.9.5
+      - safetensors==0.5.2
+      - scikit-learn==1.6.1
+      - scipy==1.13.1
+      - semantic-version==2.10.0
+      - sentencepiece==0.2.0
+      - sentry-sdk==2.20.0
+      - setproctitle==1.3.4
+      - shellingham==1.5.4
+      - simplejson==3.19.3
+      - six==1.17.0
+      - slam-llm==0.0.1
+      - smmap==5.0.2
+      - sniffio==1.3.1
+      - sortedcontainers==2.4.0
+      - soundfile==0.13.1
+      - soupsieve==2.6
+      - soxr==0.5.0.post1
+      - stack-data==0.6.3
+      - starlette==0.45.3
+      - sympy==1.13.1
+      - termcolor==2.5.0
+      - texttable==1.7.0
+      - threadpoolctl==3.5.0
+      - tiktoken==0.8.0
+      - tokenize-rt==6.1.0
+      - tokenizers==0.19.1
+      - tomli==2.2.1
+      - tomlkit==0.12.0
+      - torch==2.3.0
+      - torchaudio==2.3.0
+      - torchmetrics==1.6.1
+      - torchvision==0.15.2+cu118
+      - tqdm==4.67.1
+      - traitlets==5.14.3
+      - transformers==4.43.4
+      - triton==2.3.0
+      - typeguard==4.4.1
+      - typer==0.15.1
+      - typing-extensions==4.12.2
+      - tzdata==2025.1
+      - urllib3==2.3.0
+      - uvicorn==0.34.0
+      - wandb==0.19.6
+      - wcwidth==0.2.13
+      - websockets==12.0
+      - wetextprocessing==1.0.3
+      - wget==3.2
+      - xxhash==3.5.0
+      - yapf==0.43.0
+      - yarl==1.18.3
+      - zipp==3.21.0
+prefix: /cpfs01/shared/jishengpeng/miniconda3/envs/wavomni

Wav-Omni/examples/s2s/audio_prompt/en/prompt_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:feeda768f3add18351ee20bed5bea58f5bb0d1936bb6073b4539496e20226888
+size 303016

Wav-Omni/examples/s2s/audio_prompt/en/prompt_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:685356ad8d595caed18b836df579d29e1764327f02c740e34c166e53fe7ff7e0
+size 308662

Wav-Omni/examples/s2s/audio_prompt/en/prompt_3.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5993d88854f032e50e3bd850d6aec76222d9300d9999cec24bb4ea7186c46799
+size 303016

Wav-Omni/examples/s2s/audio_prompt/en/prompt_4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e8403303c63f3ecd78a71b423aecbff73f2c5ec50cd105f26391707ec2f7b8d
+size 319950

Wav-Omni/examples/s2s/audio_prompt/en/prompt_5.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8415294e7303851998823ea67e23d5d9033f04fd6c2170a83f3be8d1c65e7583
+size 312424

Wav-Omni/examples/s2s/audio_prompt/en/prompt_6.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17b0cbe97a1d195493cf903077a90caa50029c5a57644f60e98e4e2e547b61b
+size 335004

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:739ab2183a65576f14eec6f72ea7de6954e02014491b2876b1c528feea4ba5bc
+size 214316

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28fc8a7443a5fbadb04566e1344792fb6fc5847d27b16afeb441b813063fec58
+size 197676

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_3.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c70be19e3a974efae41a727872d2385f597f3cb48054dd2b19bd41c8e3910749
+size 198444

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa7c7ee1b196484661326e915ee3cbb5a313efdbbf67ed735ded8a0b7cd9f441
+size 216364

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_5.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8580fc452743c8fb8554b8fbf72f0962fb82346d64471c0d2f976d9abbf3f0e7
+size 219180

Wav-Omni/examples/s2s/audio_prompt/zh/prompt_6.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:751bc465053b4cdcb184cef48c3f58168a3d6c699d3b6b72230c383507d77fb8
+size 197164

Wav-Omni/examples/s2s/conf/ds_config.json

@@ -0,0 +1,19 @@
+{
+    "train_micro_batch_size_per_gpu": 4,
+    "gradient_accumulation_steps": 1,
+    "optimizer": {
+        "type": "Adam",
+        "params": {
+            "lr": 1e-4
+        }
+    },
+    "fp16": {
+        "enabled": true
+    },
+    "zero_optimization": {
+        "stage": 3,
+        "offload_optimizer": {
+            "device": "cpu"
+        }
+    }
+}

Wav-Omni/examples/s2s/conf/prompt.yaml

@@ -0,0 +1,3 @@
+dataset_config:
+    # we put prompt here, because the hydra override in shell script only support a small subset of chars
+    prompt: "Conduct a spoken conversation with the user. "

Wav-Omni/examples/s2s/conf/prompt_asr.yaml

@@ -0,0 +1,2 @@
+dataset_config:
+    prompt: "Transcribe the provided audio into accurate text. "

Wav-Omni/examples/s2s/conf/prompt_tts.yaml

@@ -0,0 +1,4 @@
+dataset_config:
+    # we put prompt here, because the hydra override in shell script only support a small subset of chars
+    # prompt: "Transcribe speech to text. Output the transcription directly without redundant content. Ensure that the output is not duplicated. "
+    prompt: "Generate a natural and expressive spoken version of the given text. "

Wav-Omni/examples/s2s/deepspeed_finetune_s2s.py

@@ -0,0 +1,47 @@
+from wavomni_llm.pipeline.finetune_deepspeed import main as train
+from wavomni_llm.utils.deepspeed_utils import deepspeed_main_wrapper
+
+import logging
+from dataclasses import dataclass, field
+from omegaconf import DictConfig, ListConfig, OmegaConf
+from s2s_config import ModelConfig, TrainConfig, DataConfig, LogConfig
+
+
+@dataclass
+class RunConfig:
+    dataset_config: DataConfig = field(default_factory=DataConfig)
+    model_config: ModelConfig = field(default_factory=ModelConfig)
+    train_config: TrainConfig = field(default_factory=TrainConfig)
+    log_config: LogConfig = field(default_factory=LogConfig)
+    debug: bool = field(default=False, metadata={"help": "Use pdb when true"})
+    metric: str = field(default="acc", metadata={"help": "The metric for evaluation"})
+    deepspeed_config: str = field(default="examples/asr_librispeech/conf/ds_config.json", metadata={"help": "The metric for evaluation"})
+
+
+@deepspeed_main_wrapper(config_name=None, version_base=None)
+def main_hydra(cfg: DictConfig):
+    run_config = RunConfig()
+    cfg = OmegaConf.merge(run_config, cfg)
+    def to_plain_list(cfg_item):
+        if isinstance(cfg_item, ListConfig):
+            return OmegaConf.to_container(cfg_item, resolve=True)
+        elif isinstance(cfg_item, DictConfig):
+            return {k: to_plain_list(v) for k, v in cfg_item.items()}
+        else:
+            return cfg_item
+    
+    # kwargs = to_plain_list(cfg)
+    kwargs = cfg
+    log_level = getattr(logging, kwargs.get("log_level", "INFO").upper())
+    
+    logging.basicConfig(level=log_level)
+    
+    if kwargs.get("debug", False):
+        import pdb;
+        pdb.set_trace()
+        
+    train(kwargs)
+
+
+if __name__ == "__main__":
+    main_hydra()

Wav-Omni/examples/s2s/demo/142001_0_rlhf.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d027ee1e2cbe76cef17ead8d8d92bfb2a68df09d756ebcce8e9a4339e8738be
+size 330796

Wav-Omni/examples/s2s/demo/142023_0_rlhf.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64278538582f31b7657b9496876a9cc5ea3c1204bfe49aaf2d085c0ca7356400
+size 221228

Wav-Omni/examples/s2s/demo/142038_0_rlhf.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3a09e765fb5e025c206ea65531143d8c60e8b545c82dc5e0d040f4e7c5a4417
+size 362540

Wav-Omni/examples/s2s/demo/64474_1_rlhf.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb2a25bb5a13844494e4e8069e277ea12aa199a6476eae06cfbfc47cf668ff64
+size 183340

Wav-Omni/examples/s2s/demo/demo_data/142038_0_rlhf.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3a09e765fb5e025c206ea65531143d8c60e8b545c82dc5e0d040f4e7c5a4417
+size 362540

Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp.jsonl

@@ -0,0 +1,4 @@
+{"key": "1", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "2", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00244-of-00325/142001_0_rlhf.wav", "source_text": "What are some things I can mix into a pizza crust?", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142001_0_rlhf.wav", "target_text": "There are lots of tasty ingredients you can mix into a pizza crust to enhance flavor and texture! You can try adding herbs like oregano, basil, or rosemary for a fragrant touch. Garlic powder or minced garlic can add a nice, strong flavor. Cheese like Parmesan or shredded mozzarella can make your crust richer and more savory. Some people also like to add a bit of sugar or honey for a slightly sweet note. You can even mix in finely chopped olives or sundried tomatoes for a unique twist. Always ensure your added ingredients are well-incorporated into the dough before baking for the best results."}
+{"key": "3", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00244-of-00325/142023_0_rlhf.wav", "source_text": "How do you take care of a frog?", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142023_0_rlhf.wav", "target_text": "Taking care of a frog involves several important steps. First, you'll need a proper habitat, like a terrarium, with enough space, humidity, and proper temperature. Ensure the terrarium has a secure lid to prevent escape. Use substrates like coconut fiber or moss to keep the environment humid. Provide hiding spots like logs or rocks for your frog to feel safe. Frogs require clean water, so provide a shallow dish with dechlorinated or bottled water. Feed your frog a diet of live insects, like crickets or worms, and dust them with a calcium supplement occasionally. Regularly clean the terrarium to prevent bacterial buildup and monitor your frog for signs of illness, consulting a vet if anything seems amiss."}
+{"key": "4", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00244-of-00325/142038_0_rlhf.wav", "source_text": "Search for a peach crumble bar recipe.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142038_0_rlhf.wav", "target_text": "Sure! A quick search online will give you some great peach crumble bar recipes. Typically, these recipes involve making a base crust, a peach filling, and a crumbly topping. You'll need ingredients like fresh peaches, flour, sugar, butter, and oats. You start by creating the crust, layering the peach filling, and finishing with the crumble topping before baking it all together. Enjoy your delicious treat!"}

Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_chinese.jsonl

@@ -0,0 +1,4 @@
+{"key": "1", "source_wav": "/cpfs01/shared/jishengpeng/Data/wavomni/tmp/73856459_1_train_3.5M_CN_ready4cosy_wo_code_switching.wav", "source_text": "回答该题：柠檬的科学名称是什么？", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "柠檬的科学名称为Citrus limon，属于芸香科柑橘属的常绿小乔木或灌木植物。它起源于亚洲南部，并在欧洲和地中海地区以及美洲热带地区广泛栽培。柠檬树可以在强烈的阳光下生长，所以它经常被栽种在阳光充足的地方，如地中海气候区。柠檬的果实是圆锥形的，皮表面略带纵行皱纹，果肉酸爽，用于制作饮料和调味料等。柠檬也是一种富含维生素C和抗氧化剂的水果，对保持健康有很大的益处。"}
+{"key": "2", "source_wav": "/cpfs01/shared/jishengpeng/Data/wavomni/tmp/96297378_1_train_3.5M_CN_ready4cosy_wo_code_switching.wav", "source_text": "请提出五种减少塑料垃圾的方法。", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142001_0_rlhf.wav", "target_text": "1. 减少使用塑料制品，转而使用可再生或可分解材料制品。2. 训练良好的垃圾分类习惯，将可回收和可分解的塑料垃圾分别分类处理。3. 限制超市、商场，以及快餐店等商家使用一次性塑料袋、餐具和纸杯等，鼓励折叠包与可重复使用的餐具。4. 呼吁政府出台相关的环境保护法律法规，限制塑料垃圾排放，处罚违规行为。5. 开展垃圾分类教育活动，吸引公众关注塑料垃圾减量，培养环保意识，减少不必要的消费行为"}
+{"key": "3", "source_wav": "/cpfs01/shared/jishengpeng/Data/wavomni/tmp/4412288_1_train_3.5M_CN_ready4cosy_wo_code_switching.wav", "source_text": "小李的薪水是每月8000元，他每个月需要交400元的养老保险和200元的医疗保险，请问他每个月实际能拿到的薪水是多少？", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142023_0_rlhf.wav", "target_text": "小李每个月的薪水是8000元，需要交400元的养老保险和200元的医疗保险。因此，他实际能拿到的薪水是多少呢？我们可以用一个简单的计算公式来计算：实际薪水 = 总薪水 - 养老保险 - 医疗保险。将小李每月的总薪水8000元代入公式，养老保险400元和医疗保险200元分别代入得：实际薪水 = 8000 - 400 - 200 = 7400元，所以小李每个月实际能拿到的薪水是7400元。"}
+{"key": "4", "source_wav": "/home/xj_data/jishengpeng/huawei/Wav-Omni/examples/s2s/demo/question_jsp.wav", "source_text": "你好，我现在是浙江大学计算机二年级的硕士生，攻读语音方向，你能给我推荐一些学习的建议吗？", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00244-of-00325/142038_0_rlhf.wav", "target_text": "没有标准回答，请随意发挥。"}

Wav-Omni/examples/s2s/demo/demo_data/jsonl_demo_jsp_train.jsonl

@@ -0,0 +1,30 @@
+{"key": "1", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "2", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "3", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "4", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "5", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "6", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "7", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "8", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "9", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "10", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "11", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "12", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "13", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "14", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "15", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "16", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "17", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "18", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "19", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "20", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "21", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "22", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "23", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "24", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "25", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "26", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "27", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "28", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "29", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}
+{"key": "30", "source_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/question_audio/train-00060-of-00325/64474_1_rlhf.wav", "source_text": "I want to help people.", "target_wav": "/cpfs01/shared/jishengpeng/Data/VoiceAssistant-400K/answer_audio/girl/train-00060-of-00325/64474_1_rlhf.wav", "target_text": "That's wonderful to hear! There are many ways you can help others. You can volunteer at local shelters, donate to charities, or simply offer support and kindness to those around you. Listening to someone's problems and providing a comforting presence can make a huge difference. Small acts of kindness, like helping a neighbor with groceries, can also have a big impact. Follow your heart and find the way that feels right for you."}

Wav-Omni/examples/s2s/demo/question_jsp.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a83d78aca6d0973e96ed1d0c59082a1f13234c9b4a80408cce29d3a9e13fb7b4
+size 248997