import pathlib import sys import os directory = pathlib.Path(os.getcwd()) sys.path.append(str(directory)) import torch import numpy as np from wav_evaluation.models.CLAPWrapper import CLAPWrapper import torch.nn.functional as F import argparse import csv from tqdm import tqdm from torch.utils.data import Dataset,DataLoader import pandas as pd import json def cal_score_by_csv(csv_path,clap_model): # audiocaps val的gt音频的clap_score计算为0.479077 input_file = open(csv_path) input_lines = input_file.readlines() clap_scores = [] caption_list,audio_list = [],[] with torch.no_grad(): for idx in tqdm(range(len(input_lines))): # text_embeddings = clap_model.get_text_embeddings([getattr(t,'caption')])# 经过了norm的embedding # audio_embeddings = clap_model.get_audio_embeddings([getattr(t,'audio_path')], resample=True) # score = clap_model.compute_similarity(audio_embeddings, text_embeddings,use_logit_scale=False) # clap_scores.append(score.cpu().numpy()) if input_lines[idx][0] == 'S': item_name, semantic = input_lines[idx].split('\t') index = item_name[2:] # import ipdb # ipdb.set_trace() caption_list.append(semantic.strip()) audio_list.append(f'/home1/liuhuadai/projects/VoiceLM-main/encodec_16k_6kbps_multiDisc/results/text_to_audio_0912/ref/{index}.wav') # import ipdb # ipdb.set_trace() if idx % 60 == 0: text_embeddings = clap_model.get_text_embeddings(caption_list)# 经过了norm的embedding audio_embeddings = clap_model.get_audio_embeddings(audio_list, resample=True)# 这一步比较耗时,读取音频并重采样到44100 score_mat = clap_model.compute_similarity(audio_embeddings, text_embeddings,use_logit_scale=False) score = score_mat.diagonal() clap_scores.append(score.cpu().numpy()) # print(caption_list) # print(audio_list) # print(score) audio_list = [] caption_list = [] # print("mean:",np.mean(np.array(clap_scores).flatten())) return np.mean(np.array(clap_scores).flatten()) [0.24463119, 0.24597324, 0.26050782, 0.25079757, 0.2501094, 0.2629509,0.25025588,0.25980043,0.27295044, 0.25655213, 0.2490872, 0.2598294,0.26491216,0.24698025,0.25086403,0.27533108,0.27969885,0.2596455,0.26313564,0.2658071] def add_clap_score_to_csv(csv_path,clap_model): df = pd.read_csv(csv_path,sep='\t') clap_scores_dict = {} with torch.no_grad(): for idx,t in enumerate(tqdm(df.itertuples()),start=1): text_embeddings = clap_model.get_text_embeddings([getattr(t,'caption')])# 经过了norm的embedding audio_embeddings = clap_model.get_audio_embeddings([getattr(t,'audio_path')], resample=True) score = clap_model.compute_similarity(audio_embeddings, text_embeddings,use_logit_scale=False) clap_scores_dict[idx] = score.cpu().numpy() df['clap_score'] = clap_scores_dict df.to_csv(csv_path[:-4]+'_clap.csv',sep='\t',index=False) if __name__ == '__main__': ckpt_path = '/home1/liuhuadai/projects/VoiceLM-main/encodec_16k_6kbps_multiDisc/useful_ckpts/CLAP' clap_model = CLAPWrapper(os.path.join(ckpt_path,'CLAP_weights_2022.pth'),os.path.join(ckpt_path,'config.yml'), use_cuda=True) clap_score = cal_score_by_csv('/home1/liuhuadai/projects/VoiceLM-main/encodec_16k_6kbps_multiDisc/Test/generate-test.txt',clap_model) out = 'text_to_audio2_0908' print(f"clap_score for {out} is:{clap_score}") print(f"clap_score for {out} is:{clap_score}") print(f"clap_score for {out} is:{clap_score}") # os.remove(csv_path)