import os import json from tqdm import tqdm from torch.utils.data import DataLoader from minigpt4.common.eval_utils import prepare_texts, init_model, eval_parser from minigpt4.conversation.conversation import CONV_VISION from minigpt4.processors.blip_processors import Blip2ImageTrainProcessor,BlipCaptionProcessor from minigpt4.datasets.datasets.video_datasets import VideoChatGPTEvalDataset,VideoChatGPTEval_consistancy,Video_validation_Dataset,TVQAEVAL,TVQAEVAL_Long parser = eval_parser() parser.add_argument("--dataset", type=str, default='msvd', help="dataset to evaluate") parser.add_argument("--add_subtitles",action='store_true',help="whether to add subtitles to the video") parser.add_argument("--name", type=str, default='3_datasets', help="evaluation name") parser.add_argument("--batch_size", type=int, default=1, help="batch size") parser.add_argument("--start", type=int, default=0, help="start from video number") parser.add_argument("--end", type=int, default=10000000, help="end at video number") args = parser.parse_args() print(args.ckpt) print(args.name) print(args.cfg_path) if "test_configs/mistral_test_config.yaml" == args.cfg_path: llm_name="mistral" else: llm_name="llama2" print("using captions",args.add_subtitles) model, vis_processor = init_model(args) conv_temp = CONV_VISION.copy() conv_temp.system = "" if args.dataset == 'video_chatgpt_generic': ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/generic_qa.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/benchmark/generic" annotations_keys=['Q','A','video_name'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'video_chatgpt_temporal': ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/temporal_qa.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/benchmark/temporal" annotations_keys=['Q','A','video_name'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'video_chatgpt_consistency': ann_path="datasets/evaluation_datasets/videochatgpt_benchmark/consistency_qa.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Test_Videos" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" annotations_keys=[['Q1','Q2'],'A','video_name'] data = VideoChatGPTEval_consistancy(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'msrvtt': ann_path="datasets/evaluation_datasets/msrvtt/val_qa_edited.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/MSRVTT/videos/all" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/msrvtt" annotations_keys=['question','answer','video_id'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'msvd': ann_path="datasets/evaluation_datasets/msvd/val_qa_edited.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/MSVD-QA/videos" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/msvd" annotations_keys=['question','answer','video_id'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'activitynet': ann_path="datasets/evaluation_datasets/activityNet/test_qa.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/Activity_net/Activity_net_videos" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles/" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/activity_net" annotations_keys=['question','answer','video_id'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=args.add_subtitles,llm_name=llm_name) elif args.dataset == 'tgif': ann_path="datasets/evaluation_datasets/tgif/Test_frameqa_question.json" videos_path="/ibex/project/c2090/datasets/VideoInstruct100K/test_videos/TGIF/mp4s" subtitles_path="/home/ataallka/minigpt_video/minigpt_multi_img/inference_subtitles" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/tgif" annotations_keys=['question','answer','gif_name'] # annotations_keys=['question','description','gif_name'] data = VideoChatGPTEvalDataset(vis_processor, videos_path, ann_path,subtitles_path,annotations_keys,videos_features_path, add_subtitles=False,llm_name=llm_name) elif args.dataset == 'tvqa': # TVQA dataset ann_path="datasets/evaluation_datasets/tvqa_short/tvqa_val.json" videos_path= "/ibex/project/c2090/datasets/TVR_dataset/videos/video_files/frames_hq/" subtitles_path="/ibex/project/c2090/datasets/TVR_dataset/TVRetrieval/data/tvqa_preprocessed_subtitles.json" videos_features_path="/ibex/project/c2106/kirolos/videos_features/evaluation/tvqa" data = TVQAEVAL(vis_processor, videos_path, ann_path,subtitles_path,videos_features_path,add_subtitles=args.add_subtitles,llm_name=llm_name) eval_dataloader = DataLoader(data, batch_size=args.batch_size, shuffle=False) minigpt4_predict = [] sub="subtitles" if args.add_subtitles else "no_subtitles" if args.start == 0 and args.end == 10000000: save_path = f'results/{args.name}_{args.dataset}_{sub}.json' else: print("start from video number",args.start) print("end at video number",args.end) save_path = f'results/{args.name}_{args.dataset}_{sub}_{args.start}_{args.end}.json' os.makedirs("results", exist_ok=True) c=0 pred_result = {} gt_result = {} if args.dataset == 'video_chatgpt_consistency': for images, texts_1,texts_2, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"): if args.start<= c = args.end : break c+=1 elif args.dataset == 'tvr': for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"): if args.start<= c = args.end : break c+=1 elif args.dataset == 'ego_schema' or args.dataset == 'tvqa' or args.dataset == 'tvqa_long_videos': for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"): if args.start<= c = args.end : break c+=1 else: for images, texts, gt_answers, lengths,videos_ids in tqdm(eval_dataloader,desc=f"Eval {args.dataset}"): if args.start<= c = args.end : break c+=1 with open(save_path, 'w') as f: json.dump(minigpt4_predict, f) print("saved results to",save_path) # save results # bleu_save_path = f'results/{args.name}_{args.dataset}_bleu.json' # cider_save_path = f'results/{args.name}_{args.dataset}_cider.json' # chatgpt_eval_save_path = f'results/{args.name}_{args.dataset}_chatgpt_eval.json' # bleu_results=eval_bleu(minigpt4_predict) # with open(bleu_save_path, 'w') as f: # json.dump(bleu_results, f) # print("bleu_results",bleu_results) # cider_results=eval_cider(pred_result,gt_result) # with open(cider_save_path, 'w') as f: # json.dump(cider_results, f) # print("mean_cider_scores:",cider_results['mean_cider_scores']) # chatgpt_results=chat_gpt_eval(pred_result,gt_result) # with open(chatgpt_eval_save_path, 'w') as f: # json.dump(chatgpt_results, f) # print("avg_chatgpt_score",chatgpt_results['avg_chatgpt_score']) # print(chatgpt_results)