Upload 31 files

video_regression_script (1).py

@@ -0,0 +1,191 @@
+'''
+python video_regression_script.py "" xiangling_mp4_dir_tiny
+
+1. **visual quality**:
+   - **涵义**: 视觉质量
+   - **解释**: 视频在清晰度、分辨率、亮度和色彩等方面的质量。这个维度评估视频的视觉表现，包括图像的清晰度、色彩的准确性和整体的视觉吸引力。
+
+2. **temporal consistency**:
+   - **涵义**: 时间一致性
+   - **解释**: 视频中物体或人物的一致性。这个维度评估视频在时间上的连贯性，即视频中的物体或人物在不同帧之间是否保持一致，没有明显的跳跃或不连贯的现象。
+
+3. **dynamic degree**:
+   - **涵义**: 动态程度
+   - **解释**: 视频中动态变化的程度。这个维度评估视频的动态性，即视频中物体或场景的变化程度，包括运动的频率和幅度。
+
+4. **text-to-video alignment**:
+   - **涵义**: 文本与视频的对齐
+   - **解释**: 文本提示与视频内容之间的对齐程度。这个维度评估视频内容与给定文本提示之间的匹配程度，即视频是否准确地反映了文本提示所描述的内容。
+
+5. **factual consistency**:
+   - **涵义**: 事实一致性
+   - **解释**: 视频内容与常识和事实知识的一致性。这个维度评估视频内容是否符合常识和事实知识，即视频中的内容是否真实可信，没有明显的逻辑错误或与现实不符的情况。
+
+import pandas as pd
+edf = pd.read_csv("evaluation_results.csv")
+edf.describe()
+
+print(edf.sort_values(by = "temporal consistency", ascending = True).head(5).to_markdown())
+
+|    | video_name                                                                                         |   visual quality |   temporal consistency |   dynamic degree |   text-to-video alignment |   factual consistency |
+|---:|:---------------------------------------------------------------------------------------------------|-----------------:|-----------------------:|-----------------:|--------------------------:|----------------------:|
+| 26 | solo,Xiangling,_shave_with_a_razor__genshin_impact__,1girl,highres,_seed_3140464511.mp4            |             2.8  |                   1.14 |             2.97 |                      2.78 |                  1.26 |
+|  0 | solo,Xiangling,_carry_money_in_a_wallet__genshin_impact__,1girl,highres,_seed_1294598571.mp4       |             2.69 |                   1.2  |             2.88 |                      2.7  |                  1.34 |
+|  9 | solo,Xiangling,_sweep_dust_with_a_broom__genshin_impact__,1girl,highres,_seed_3483804345.mp4       |             2.72 |                   1.22 |             2.89 |                      2.86 |                  1.2  |
+| 25 | solo,Xiangling,_brush_teeth_with_a_toothbrush__genshin_impact__,1girl,highres,_seed_2612536091.mp4 |             2.75 |                   1.23 |             2.91 |                      2.67 |                  1.44 |
+| 14 | solo,Xiangling,_store_trash_in_a_bag__genshin_impact__,1girl,highres,_seed_4130052080.mp4          |             2.72 |                   1.25 |             2.86 |                      2.77 |                  1.27 |
+'''
+
+import os
+import time
+import json
+import numpy as np
+import av
+import torch
+from PIL import Image
+import functools
+from transformers import AutoProcessor, AutoConfig
+from models.idefics2 import Idefics2ForSequenceClassification
+from models.conversation import conv_templates
+from typing import List
+import csv
+import argparse
+from tqdm import tqdm
+import shutil
+
+# 初始化模型和处理器
+processor = AutoProcessor.from_pretrained("TIGER-Lab/VideoScore")
+model = Idefics2ForSequenceClassification.from_pretrained("TIGER-Lab/VideoScore", torch_dtype=torch.bfloat16).eval()
+
+MAX_NUM_FRAMES = 24
+conv_template = conv_templates["idefics_2"]
+
+VIDEO_EVAL_PROMPT = """
+Suppose you are an expert in judging and evaluating the quality of AI-generated videos, 
+please watch the following frames of a given video and see the text prompt for generating the video, 
+then give scores from 5 different dimensions:
+(1) visual quality: the quality of the video in terms of clearness, resolution, brightness, and color
+(2) temporal consistency, the consistency of objects or humans in video
+(3) dynamic degree, the degree of dynamic changes
+(4) text-to-video alignment, the alignment between the text prompt and the video content
+(5) factual consistency, the consistency of the video content with the common-sense and factual knowledge
+
+For each dimension, output a number from [1,2,3,4], 
+in which '1' means 'Bad', '2' means 'Average', '3' means 'Good', 
+'4' means 'Real' or 'Perfect' (the video is like a real video)
+Here is an output example:
+visual quality: 4
+temporal consistency: 4
+dynamic degree: 3
+text-to-video alignment: 1
+factual consistency: 2
+
+For this video, the text prompt is "{text_prompt}",
+all the frames of video are as follows: 
+
+"""
+
+aspect_mapping = [
+    "visual quality",
+    "temporal consistency",
+    "dynamic degree",
+    "text-to-video alignment",
+    "factual consistency",
+]
+
+def score(prompt: str, images: List[Image.Image]):
+    if not prompt:
+        raise ValueError("Please provide a prompt")
+    model.to("cuda")
+    if not images:
+        images = None
+
+    flatten_images = []
+    for x in images:
+        if isinstance(x, list):
+            flatten_images.extend(x)
+        else:
+            flatten_images.append(x)
+
+    flatten_images = [Image.open(x) if isinstance(x, str) else x for x in flatten_images]
+    inputs = processor(text=prompt, images=flatten_images, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    logits = outputs.logits
+    num_aspects = logits.shape[-1]
+    aspects = [aspect_mapping[i] for i in range(num_aspects)]
+
+    aspect_scores = {}
+    for i, aspect in enumerate(aspects):
+        aspect_scores[aspect] = round(logits[0, i].item(), 2)
+    return aspect_scores
+
+def read_video_pyav(container, indices):
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+def eval_video(prompt, video: str):
+    container = av.open(video)
+
+    total_frames = container.streams.video[0].frames
+    if total_frames > MAX_NUM_FRAMES:
+        indices = np.arange(0, total_frames, total_frames / MAX_NUM_FRAMES).astype(int)
+    else:
+        indices = np.arange(total_frames)
+    video_frames = read_video_pyav(container, indices)
+
+    frames = [Image.fromarray(x) for x in video_frames]
+
+    eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
+
+    num_image_token = eval_prompt.count("<image>")
+    if num_image_token < len(frames):
+        eval_prompt += "<image> " * (len(frames) - num_image_token)
+
+    aspect_scores = score(eval_prompt, [frames])
+    return aspect_scores
+
+def main():
+    parser = argparse.ArgumentParser(description="Evaluate videos in a directory.")
+    parser.add_argument("prompt", type=str, help="Text prompt for the video evaluation.")
+    parser.add_argument("video_dir", type=str, help="Directory containing video files.")
+    args = parser.parse_args()
+
+    video_files = [os.path.join(args.video_dir, f) for f in os.listdir(args.video_dir) if f.endswith(('.mp4', '.avi', '.mkv'))]
+
+    # 创建五个指标对应的文件夹
+    output_dirs = {aspect: f"{aspect}_videos" for aspect in aspect_mapping}
+    for dir_name in output_dirs.values():
+        os.makedirs(dir_name, exist_ok=True)
+
+    with open("evaluation_results.csv", "w", newline='') as csvfile:
+        fieldnames = ["video_name", "visual quality", "temporal consistency", "dynamic degree", "text-to-video alignment", "factual consistency"]
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer.writeheader()
+
+        for video_file in tqdm(video_files, desc="Evaluating videos"):
+            video_name = os.path.basename(video_file)
+            aspect_scores = eval_video(args.prompt, video_file)
+            aspect_scores["video_name"] = video_name
+            writer.writerow(aspect_scores)
+
+            # 将视频文件复制到对应的文件夹中，并以指标值为名称保存
+            for aspect, score in aspect_scores.items():
+                if aspect != "video_name":
+                    score_str = f"{score:.2f}".replace('.', '_')  # 将小数点替换为下划线以便于排序
+                    new_video_name = f"{score_str}_{video_name}"
+                    output_dir = output_dirs[aspect]
+                    shutil.copy(video_file, os.path.join(output_dir, new_video_name))
+
+if __name__ == "__main__":
+    main()

xiangling_mp4_dir_tiny/solo,Xiangling,_bake_meat_in_an_oven__genshin_impact__,1girl,highres,_seed_2952770938.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1b459a6a0f454fc18273d5bc8580f8f8b5c3b81126f94b78131783d949a4c0b
+size 373772

xiangling_mp4_dir_tiny/solo,Xiangling,_brush_teeth_with_a_toothbrush__genshin_impact__,1girl,highres,_seed_2612536091.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff4ad026844e136f05f5bef06882751d7bf0c797a4e4ab4397a41113a4ee23a
+size 438596

xiangling_mp4_dir_tiny/solo,Xiangling,_carry_money_in_a_wallet__genshin_impact__,1girl,highres,_seed_1294598571.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef071a84a9de6ab348d54916c8f8ce3a8ed63e7cd6d4ffa4f3efa5e53fcc3b7a
+size 473907

xiangling_mp4_dir_tiny/solo,Xiangling,_change_channels_with_a_remote_control__genshin_impact__,1girl,highres,_seed_900516482.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:928e13e88855b452bdeb4cae289caf4beccee5305f931f1456c53d25acbf6551
+size 331714

xiangling_mp4_dir_tiny/solo,Xiangling,_check_health_status_with_a_thermometer__genshin_impact__,1girl,highres,_seed_3530484100.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef3253a61aafad5b44c8d496fadc474084f335d026fbb573e34f42d340456d32
+size 364511

xiangling_mp4_dir_tiny/solo,Xiangling,_clean_the_floor_with_a_vacuum_cleaner__genshin_impact__,1girl,highres,_seed_3041366259.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98165eec3b499e026b05984d968e60bdd6dfc534b53bf82773d6a1213f6a387c
+size 277776

xiangling_mp4_dir_tiny/solo,Xiangling,_cut_meat_with_a_knife__genshin_impact__,1girl,highres,_seed_3022493544.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b52b820da4979b62008ec41892673beb0390563251442350652008d5f5ac0d5
+size 405400

xiangling_mp4_dir_tiny/solo,Xiangling,_dry_hair_with_a_towel__genshin_impact__,1girl,highres,_seed_4154310316.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fef1d282f6ae012591af1f3f6e4b151f7027bbe427659d2f6c8b0f81019b545
+size 323404

xiangling_mp4_dir_tiny/solo,Xiangling,_look_in_a_mirror__genshin_impact__,1girl,highres,_seed_3970688016.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f0c4b48d7b9778bb69a6dc629951b0ef89478162b766cd4e85ef0635d4d34b3
+size 293939

xiangling_mp4_dir_tiny/solo,Xiangling,_organize_documents_in_a_folder__genshin_impact__,1girl,highres,_seed_3088899046.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2910d2899aa2420106dd6e772f2c7bfb31b6eccdafff2eeaf951843cde33d7a3
+size 350121

xiangling_mp4_dir_tiny/solo,Xiangling,_pack_clothes_in_a_bag__genshin_impact__,1girl,highres,_seed_1830441848.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13736b35595c67830eec7c80b3b52b29a0c5648614ad27accc38e4bc359cc83b
+size 296951

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_dessert_on_a_plate__genshin_impact__,1girl,highres,_seed_2261399072.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8de65d0143218b0f527a8d3f096509997bd218b6e0c17937710e75a3a9f07842
+size 402652

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_milk_in_a_cup__genshin_impact__,1girl,highres,_seed_2726786606.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2485220e66b971535b2ee1caacd157220f0b27a72db050d3e0d5a8664ca47f6e
+size 381171

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_rice_on_a_plate__genshin_impact__,1girl,highres,_seed_1973143025.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a97b98a60737201fa957f4f03144178b930b238a892b65353902e17889233886
+size 353967

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_salad_in_a_bowl__genshin_impact__,1girl,highres,_seed_3262702916.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca26ea5eed4b9e7b6228b231c66cd1745f3ab10a4b7633db14f8dbf067b1804
+size 324734

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_soup_on_a_plate__genshin_impact__,1girl,highres,_seed_2874330799.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:856da708b3df1f1a0bc8f23ed800e7774cc83f650500b60a96b2ff1f3aa5cd9b
+size 303181

xiangling_mp4_dir_tiny/solo,Xiangling,_serve_tea_in_a_cup__genshin_impact__,1girl,highres,_seed_371786328.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c45b988bfc0f3126f60f9d207b9ac9fb5715093c57152151e4f3c4dc804613
+size 306113

xiangling_mp4_dir_tiny/solo,Xiangling,_shade_from_the_sun_with_an_umbrella__genshin_impact__,1girl,highres,_seed_3322535967.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b44cf1ffe09065c9ac8de8a680055a5b0d714ad4d18602dcf7026010f115d10
+size 381418

xiangling_mp4_dir_tiny/solo,Xiangling,_shave_with_a_razor__genshin_impact__,1girl,highres,_seed_3140464511.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9579afb38d86349722439e1824cf7ef1d134398560b8fbf7856d8bcb50cc93e
+size 489060

xiangling_mp4_dir_tiny/solo,Xiangling,_stew_soup_in_a_pot__genshin_impact__,1girl,highres,_seed_94601712.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faf86f86ec237f2ec410641880cd283e6a53c187f6bb24e56543901c5ef0d03d
+size 469362

xiangling_mp4_dir_tiny/solo,Xiangling,_store_candy_in_a_bag__genshin_impact__,1girl,highres,_seed_335895722.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dd3943e096b64f7d872ce592ef082be1b20f3de1c293a151245e39b6ead9856
+size 373653

xiangling_mp4_dir_tiny/solo,Xiangling,_store_flowers_in_a_basket__genshin_impact__,1girl,highres,_seed_2209790483.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:340a0c4d5d36ac6d9025a1f2442255a881a9c423bd36edfd25d50a8af62ee2c4
+size 462308

xiangling_mp4_dir_tiny/solo,Xiangling,_store_fruits_in_a_basket__genshin_impact__,1girl,highres,_seed_3894895115.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a0e6864264f5d082c00de96eace11d99bf6109e63b05663b71e8c1c3b6dd217
+size 394510

xiangling_mp4_dir_tiny/solo,Xiangling,_store_gifts_in_a_box__genshin_impact__,1girl,highres,_seed_2436852429.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff928bdac7e2aaea6f716671d3d7e8cf5ac7d28ef9336a4561d306b233267c65
+size 384550

xiangling_mp4_dir_tiny/solo,Xiangling,_store_jam_in_a_jar__genshin_impact__,1girl,highres,_seed_4278438878.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5db3b5c444292f1f8f142c66b98225c933dfeb0ac6a29a0a43928809a67ec3a3
+size 325563

xiangling_mp4_dir_tiny/solo,Xiangling,_store_trash_in_a_bag__genshin_impact__,1girl,highres,_seed_4130052080.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee5c93ca7504c60b90333ef9765a8c6ddbd954183a87ab3d293a088481b75c8e
+size 423134

xiangling_mp4_dir_tiny/solo,Xiangling,_store_water_in_a_bottle__genshin_impact__,1girl,highres,_seed_2864860181.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9628aed6e8985a0fa1783b1c4f131890b3e8b5414f258a21bde784b8b019200e
+size 463397

xiangling_mp4_dir_tiny/solo,Xiangling,_store_wine_in_a_bottle__genshin_impact__,1girl,highres,_seed_218350238.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2d9c62aca23f2683573563153322c48477eaf6d82cac86722c853168521e1c5
+size 306169

xiangling_mp4_dir_tiny/solo,Xiangling,_sweep_dust_with_a_broom__genshin_impact__,1girl,highres,_seed_3483804345.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86765a81335329e90ec0527874a0fbafdcbf1dcb71878c647a03d4306c18f9e6
+size 382282

xiangling_mp4_dir_tiny/solo,Xiangling,_take_photos_with_a_camera__genshin_impact__,1girl,highres,_seed_2687423072.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4b1122c591b5bde3351e3502442cbbb9138ceb5a20f5326aa944be53612a8be
+size 352669