initial files upload

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+files/*.mp4 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.DS_Store
+*.gif
+gradio_cached_examples/

app.py

@@ -0,0 +1,114 @@
+import os
+import gradio as gr
+import pandas as pd
+
+
+BASELINE = f'<a target="_blank" href=https://github.com/showlab/loveu-tgve-2023 style="color: blue; text-decoration: underline;text-decoration-style: dotted;">Tune-A-Video (Baseline)</a>'
+COLS = ["Method", "CLIPScore (Frame Consistency) ⬆️", "CLIPScore (Text Alignment) ⬆️", "PickScore ⬆️", "Human Preference ⬆️", "References"]
+TYPES = ["markdown", "number", "number", "number", "str", "markdown"]
+
+
+def get_leaderboard():
+    all_data = []
+    
+    baseline_0 = {
+        "Method": '**Tune-A-Video**',
+        "CLIPScore (Frame Consistency) ⬆️":0.92,
+        "CLIPScore (Text Alignment) ⬆️":27.12,
+        "PickScore ⬆️":20.36,
+        "Human Preference ⬆️":'',
+        "References": ','.join([f'<a target="_blank" href="https://arxiv.org/abs/2212.11565" style="color: blue">Paper</a>', 
+                               f'<a target="_blank" href="https://github.com/showlab/Tune-A-Video" style="color: blue">Code</a>', 
+                               f'<a target="_blank" href="https://tuneavideo.github.io/" style="color: blue">Website</a>', 
+                               f'<a target="_blank" href="https://huggingface.co/spaces/Tune-A-Video-library/Tune-A-Video-inference" style="color: blue">Demo</a>'])
+    }
+    baseline_1 = {
+        "Method": 'VideoCrafter (todo)',
+        "References": ','.join([f'<a target="_blank" href="https://github.com/VideoCrafter/VideoCrafter" style="color: blue">Code</a>',
+                                 f'<a target="_blank" href="https://huggingface.co/spaces/VideoCrafter/VideoCrafter" style="color: blue">Demo</a>'])
+    }
+    all_data += [baseline_0, baseline_1]
+
+    dataframe = pd.DataFrame.from_records(all_data)
+    dataframe = dataframe.sort_values(by=['PickScore ⬆️'], ascending=False)
+    print(dataframe)
+    dataframe = dataframe[COLS]
+    return dataframe
+
+leaderboard = get_leaderboard()
+    
+def refresh():
+    return get_leaderboard()
+
+def load_edited_video(source_video, *args):
+    result = source_video.split('/')[-1].split('.mp4')[0] + '-edit.mp4'
+    return os.path.join(os.path.dirname(__file__), f"files/{result}")
+
+
+block = gr.Blocks()
+with block:
+    with gr.Tab("Leaderboard"): 
+        with gr.Row():
+            gr.Markdown(f"""
+    # 🤗 LOVEU-TGVE @ CVPR 2023 Leaderboard
+    <font size="4">
+    <b>Welcome to the <a href="https://sites.google.com/view/loveucvpr23/track4" target="_blank">Text-Guided Video Editing (TGVE)</a> competition leaderboard of <a href="https://sites.google.com/view/loveucvpr23/home" target="_blank">LOVEU Workshop @ CVPR 2023</a>!</b>
+
+    Leveraging AI for video editing has the potential to unleash creativity for artists across all skill levels. The rapidly-advancing field of Text-Guided Video Editing (TGVE) is here to address this challenge. Recent works in this field include <a href="https://tuneavideo.github.io/" target="_blank">Tune-A-Video</a>, <a href="https://research.runwayml.com/gen2" target="_blank">Gen-2</a>, and <a href="https://dreamix-video-editing.github.io/" target="_blank">Dreamix</a>. 
+    In this competition track, we provide a standard set of videos and prompts. As a researcher, you will develop a model that takes a video and a prompt for how to edit it, and your model will produce an edited video. For instance, you might be given a video of “a man is surfing inside the barrel of a wave,” and your model will edit the video to “a man is surfing on a wave made of aurora borealis.”
+
+    During the competition, evaluation results performed against the following 3 automatic metrics will be displayed on the leaderboard:
+    - <a href="https://arxiv.org/abs/2103.00020" target="_blank">CLIPScore</a> (Frame Consistency) - the average cosine similarity between all pairs of CLIP image embeddings computed on all frames of output videos.
+    - <a href="https://arxiv.org/abs/2103.00020" target="_blank">CLIPScore</a> (Text Alignment) - the average CLIP score between all frames of output videos and corresponding edited prompts.
+    - <a href="https://arxiv.org/abs/2305.01569" target="_blank">PickScore</a> - the average PickScore between all frames of output videos.
+
+    After all submissions are uploaded, we will run a human-evaluation of all submitted videos. Specifically, we will have human labelers compare all submitted videos. Labelers will evaluate videos on the following criteria:
+
+    - Text alignment: How well does the generated video match the caption?
+    - Structure: How well does the generated video preserve the structure of the original video?
+    - Quality: Aesthetically, how good is this video?
+
+    We will choose a winner and a runner-up based on the human evaluation results. 
+    </font>
+
+    The **bold** method name indicates that the implementation is **official** (by the author / developer of the original method).""")
+        
+        with gr.Row():
+            leaderboard_table = gr.components.Dataframe(value=leaderboard, headers=COLS,
+                                                        datatype=TYPES, max_rows=10)
+        with gr.Row():
+            refresh_button = gr.Button("Refresh")
+            refresh_button.click(refresh, inputs=[], outputs=[leaderboard_table]) 
+        block.load(refresh, inputs=[], outputs=[leaderboard_table])
+
+    with gr.Tab("Baseline Demo"): 
+        with gr.Row():
+            gr.Markdown(f"""Some examples generated by {BASELINE} are shown below.""")
+        with gr.Row():
+            with gr.Column():
+                source_video = gr.Video(type="file", label='Source Video', format="mp4", interactive=True)
+                source_prompt = gr.Textbox(label='Source Prompt',
+                                # info='A good prompt describes each frame and most objects in video. Especially, it has the object or attribute that we want to edit or preserve.',
+                                max_lines=2,
+                                placeholder='Example: "A cat in the grass in the sun."',
+                                # value='A cat in the grass in the sun.'
+                                )
+
+            with gr.Column():
+                result = gr.Video(type="file", label='Edited Video', format="mp4", interactive=True)
+                editing_prompt = gr.Textbox(label='Editing Prompt',
+                                    # info='A reasonable composition of video may achieve better results(e.g., "sunflower" video with "Van Gogh" prompt is better than "sunflower" with "Monet")',
+                                    max_lines=2,
+                                    placeholder='Example: "A dog in the grass in the sun."',
+                                    # value='A dog in the grass in the sun.'
+                                    )
+        
+        with gr.Row():
+            from example import examples
+            gr.Examples(examples=examples,
+                    inputs=[source_video, source_prompt, editing_prompt],
+                    outputs=result,
+                    fn=load_edited_video,
+                    cache_examples=True,
+                    )
+block.launch()

example.py

@@ -0,0 +1,35 @@
+import os
+
+
+examples = [
+    [os.path.join(os.path.dirname(__file__), "files/cat-in-the-sun.mp4"), 
+     'A cat in the grass in the sun.', 
+     'A dog in the grass in the sun.'],
+    [os.path.join(os.path.dirname(__file__), "files/surfer-on-wave.mp4"),
+     'A man is surfing inside the barrel of a wave.',
+     'A man is surfing on a wave made of aurora borealis.'],
+    [os.path.join(os.path.dirname(__file__), "files/red-roses-sunny-day.mp4"),
+     'A static shot of red roses in sunlight, gently swaying in the breeze.',
+     'A static shot of red roses in sunlight, gently swaying in the breeze, origami style.'],
+    [os.path.join(os.path.dirname(__file__), "files/earth-full-view.mp4"),
+     'Full view of the Earth as it moves slowly toward the sun.',
+     'Full view of the Earth as it moves slowly through a fireworks display.'],
+    [os.path.join(os.path.dirname(__file__), "files/ship-sailing.mp4"),
+     'A ship sails on the sea during sunset.',
+     'A ship sails on the sea during sunset, 2D vector art.'],
+    [os.path.join(os.path.dirname(__file__), "files/sunset-beach-yoga.mp4"),
+     'A woman does yoga on the beach during sunset.',
+     'A panda does yoga on the beach during sunset.'],
+    [os.path.join(os.path.dirname(__file__), "files/las-vegas-time-lapse.mp4"),
+     'A mesmerizing time-lapse showcasing iconic architectures amidst the cityscape of Las Vegas in night.',
+     'A mesmerizing time-lapse showcasing iconic architectures amidst the cityscape of New York in night.'],
+    [os.path.join(os.path.dirname(__file__), "files/gold-fish.mp4"),
+     'Several goldfish swim in a tank.',
+     'Several sharks swim in a tank.'],
+    [os.path.join(os.path.dirname(__file__), "files/bird-on-feeder.mp4"),
+     'A brown bird sits on a feeder that is being hung by a red string.',
+     'A brown squirrel sits on a feeder that is being hung by a red string.'],
+    [os.path.join(os.path.dirname(__file__), "files/sharks-swimming.mp4"),
+     'Two grey sharks swim in the blue ocean on a coral reef.',
+     'Two quadrotor drones swim in the blue ocean on a coral reef.'],
+]

files/bird-on-feeder-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b22afb38ddaa80225e267bff0fcef7eaf03a4016fc1d20206189758a1bcd7d9
+size 2801107

files/bird-on-feeder.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c288cbd85064510bd64bc5dfe0a3adafa2bfd025b0012614454d77d3d319df5
+size 1345796

files/cat-in-the-sun-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dae8f5493ea977623f1b8013c24e739c525126408d93dcb0a2ca66c42507dff
+size 3281141

files/cat-in-the-sun.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89e90b0c23da0a5d3f66edb55befaa51bfa579d215c46a88cf9a4731e8841e57
+size 1215272

files/earth-full-view-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7348332fe5d0d63e1cf5819643c8ec4186a6e2e412009528877f9b68187a0bd4
+size 3279505

files/earth-full-view.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:350b59f1ac6f9efb03718de67002256bc75155fe4e819e1c359e326fc4a96b64
+size 1186554

files/gold-fish-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:938e27894d3c9d2281d09d84e9bed9faa40f9141d8059e00f7d07170a99acf1f
+size 3240881

files/gold-fish.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90f0a1c012fc8e7db252409da3361bdeb999f138057279027d4d13a911ceb05a
+size 1236588

files/las-vegas-time-lapse-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e1ba6e83ffaf27243c6b61d4f2ea50b77b80dea3f4fbaa2f00c0b138f2a671f
+size 2670173

files/las-vegas-time-lapse.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8508c6832cbd4391082e4a6d5e6fad1745ad0dae14fac579c864f380ce75da0a
+size 1264239

files/red-roses-sunny-day-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e95a95e6dc4956c110fbe3540094e820c7a58bae4c1c9ff31b253b2eca3a7b4
+size 3131316

files/red-roses-sunny-day.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:614826f410c8c0d6366b5c873051f1f38abb11dc6a128c9a414d728eeed139de
+size 1232182

files/sharks-swimming-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a29a5d592b4374c3baba596b71ed10d261b7b5e303f262cc27c2d751cb764d3
+size 3320788

files/sharks-swimming.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67e62534642dcf9f94873b3bfbf9d6ec58c6da934885997e27afe89c32f7cc3b
+size 1750076

files/ship-sailing-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c32e565e237276d9f9ba70db987fe73691e157847500601b0b0a35a23cfeeee
+size 1925050

files/ship-sailing.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81198e3f149e460da1233bca3137dd03c563953b148d5031fcd8809ff47638f0
+size 1729069

files/sunset-beach-yoga-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ef8c94a5534e2b72fb4d2cab3db82ec2c811e343983ec01eb2ab7905e6fa72d
+size 2672716

files/sunset-beach-yoga.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53c9baa98ddfc60422dddb415a10f0a99d7d3dc7eee559910cd410717883e41a
+size 1559230

files/surfer-on-wave-edit.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71ee8e0fba75de4953d6f25c78cc98ef29f896dd49c65b4e8f2f88335950597a
+size 2700156

files/surfer-on-wave.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd58f430e613e28139258a62d8df11a17320c9d7e1fb3d0499de39ac587be845
+size 1215909

requirements.txt

@@ -0,0 +1,6 @@
+apscheduler==3.10.1
+gradio==3.30.0
+huggingface_hub==0.14.1
+numpy==1.24.2
+pandas==2.0.1
+transformers==4.29.1