Spaces:
Runtime error
Runtime error
Mateo Fidabel
commited on
Commit
•
6f417f5
1
Parent(s):
e6915e1
Added more examples, added about info
Browse files- app.py +33 -6
- examples/condition_image_4.png +0 -0
- examples/condition_image_5.png +0 -0
- examples/condition_image_6.png +0 -0
- examples/condition_image_7.png +0 -0
app.py
CHANGED
@@ -26,16 +26,39 @@ p_params = replicate(params)
|
|
26 |
title = "# 🧨 ControlNet on Segment Anything 🤗"
|
27 |
description = """This is a demo on 🧨 ControlNet based on Meta's [Segment Anything Model](https://segment-anything.com/).
|
28 |
|
29 |
-
Upload a Segment Anything Segmentation Map, write a prompt, and generate images 🤗 This demo is still Work in Progress, so don't expect it to work well for now !!
|
30 |
|
31 |
-
|
32 |
-
Test some of the examples below to give it a try ⬇️
|
33 |
"""
|
34 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
35 |
examples = [["contemporary living room of a house", "low quality", "examples/condition_image_1.png"],
|
36 |
["new york buildings, Vincent Van Gogh starry night ", "low quality, monochrome", "examples/condition_image_2.png"],
|
37 |
-
["contemporary living room, high quality, 4k, realistic", "low quality, monochrome, low res", "examples/condition_image_3.png"]
|
|
|
|
|
|
|
|
|
38 |
|
|
|
39 |
|
40 |
# Inference Function
|
41 |
def infer(prompts, negative_prompts, image, num_inference_steps = 50, seed = 4, num_samples = 4):
|
@@ -93,7 +116,7 @@ prompt = gr.Textbox(lines=1, label="Prompt", value=default_example[0])
|
|
93 |
negative_prompt = gr.Textbox(lines=1, label="Negative Prompt", value=default_example[1])
|
94 |
|
95 |
|
96 |
-
with gr.Blocks(css=
|
97 |
with gr.Row():
|
98 |
with gr.Column():
|
99 |
# Title
|
@@ -103,10 +126,12 @@ with gr.Blocks(css="h1 { text-align: center }") as demo:
|
|
103 |
|
104 |
with gr.Column():
|
105 |
# Examples
|
|
|
106 |
gr.Examples(examples=examples,
|
107 |
inputs=[prompt, negative_prompt, cond_img],
|
108 |
outputs=output,
|
109 |
-
fn=infer
|
|
|
110 |
|
111 |
# Images
|
112 |
with gr.Row(variant="panel"):
|
@@ -130,6 +155,8 @@ with gr.Blocks(css="h1 { text-align: center }") as demo:
|
|
130 |
submit = gr.Button("Generate")
|
131 |
# TODO: Download Button
|
132 |
|
|
|
|
|
133 |
|
134 |
submit.click(infer,
|
135 |
inputs=[prompt, negative_prompt, cond_img, num_steps, seed, num_samples],
|
|
|
26 |
title = "# 🧨 ControlNet on Segment Anything 🤗"
|
27 |
description = """This is a demo on 🧨 ControlNet based on Meta's [Segment Anything Model](https://segment-anything.com/).
|
28 |
|
29 |
+
Upload a Segment Anything Segmentation Map, write a prompt, and generate images 🤗 This demo is still a Work in Progress, so don't expect it to work well for now !!
|
30 |
|
31 |
+
⌛️ It takes about 30~ seconds to generate 4 samples, to get faster results, don't forget to reduce the Nº Samples to 1.
|
|
|
32 |
"""
|
33 |
|
34 |
+
about = """
|
35 |
+
|
36 |
+
|
37 |
+
# 👨💻 About the model
|
38 |
+
|
39 |
+
This model is based on the [ControlNet Model](https://huggingface.co/blog/controlnet), which allow us to generate Images using some sort of condition image. For this model, we selected the segmentation maps produced by Meta's new segmentation model called [Segment Anything Model](https://github.com/facebookresearch/segment-anything) as the condition image. We then trained the model to generate images based on the structure of the segmentation maps and the text prompts given.
|
40 |
+
|
41 |
+
|
42 |
+
# 💾 About the dataset
|
43 |
+
|
44 |
+
For the training, we generated a segmented dataset based on the [COYO-700M](https://huggingface.co/datasets/kakaobrain/coyo-700m) dataset. The dataset provided us with the images, and the text prompts. For the segmented images, we used [Segment Anything Model](https://github.com/facebookresearch/segment-anything). We then created 8k samples train our model on, which isn't a lot, but as a team, we have been very busy with many other responsibilities and time constraints, which made it challenging to dedicate a lot of time to generating a larger dataset. Despite the constraints we faced, we have still managed to achieve some nice results 🙌
|
45 |
+
|
46 |
+
You can check the generated datasets below ⬇️
|
47 |
+
- [sam-coyo-2k](https://huggingface.co/datasets/mfidabel/sam-coyo-2k)
|
48 |
+
- [sam-coyo-2.5k](https://huggingface.co/datasets/mfidabel/sam-coyo-2.5k)
|
49 |
+
- [sam-coyo-3k](https://huggingface.co/datasets/mfidabel/sam-coyo-3k)
|
50 |
+
|
51 |
+
"""
|
52 |
+
|
53 |
examples = [["contemporary living room of a house", "low quality", "examples/condition_image_1.png"],
|
54 |
["new york buildings, Vincent Van Gogh starry night ", "low quality, monochrome", "examples/condition_image_2.png"],
|
55 |
+
["contemporary living room, high quality, 4k, realistic", "low quality, monochrome, low res", "examples/condition_image_3.png"],
|
56 |
+
["internal stairs of a japanese house", "low quality, low res, people, kids", "examples/condition_image_4.png"],
|
57 |
+
["a photo of a girl taking notes", "low quality, low res, painting", "examples/condition_image_5.png"],
|
58 |
+
["painting of an hot air ballon flying over a valley, The Great Wave off Kanagawa style, blue and white colors", "low quality, low res", "examples/condition_image_6.png"],
|
59 |
+
["painting of families enjoying the sunset, The Garden of Earthly Delights style, joyful", "low quality, low res", "examples/condition_image_7.png"]]
|
60 |
|
61 |
+
css = "h1 { text-align: center } .about { text-align: justify; padding-left: 10%; padding-right: 10%; }"
|
62 |
|
63 |
# Inference Function
|
64 |
def infer(prompts, negative_prompts, image, num_inference_steps = 50, seed = 4, num_samples = 4):
|
|
|
116 |
negative_prompt = gr.Textbox(lines=1, label="Negative Prompt", value=default_example[1])
|
117 |
|
118 |
|
119 |
+
with gr.Blocks(css=css) as demo:
|
120 |
with gr.Row():
|
121 |
with gr.Column():
|
122 |
# Title
|
|
|
126 |
|
127 |
with gr.Column():
|
128 |
# Examples
|
129 |
+
gr.Markdown("Try some of the examples below ⬇️")
|
130 |
gr.Examples(examples=examples,
|
131 |
inputs=[prompt, negative_prompt, cond_img],
|
132 |
outputs=output,
|
133 |
+
fn=infer,
|
134 |
+
examples_per_page=4)
|
135 |
|
136 |
# Images
|
137 |
with gr.Row(variant="panel"):
|
|
|
155 |
submit = gr.Button("Generate")
|
156 |
# TODO: Download Button
|
157 |
|
158 |
+
with gr.Row():
|
159 |
+
gr.Markdown(about, elem_classes="about")
|
160 |
|
161 |
submit.click(infer,
|
162 |
inputs=[prompt, negative_prompt, cond_img, num_steps, seed, num_samples],
|
examples/condition_image_4.png
ADDED
![]() |
examples/condition_image_5.png
ADDED
![]() |
examples/condition_image_6.png
ADDED
![]() |
examples/condition_image_7.png
ADDED
![]() |