Add files

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "CutLER"]
+	path = CutLER
+	url = https://github.com/facebookresearch/CutLER

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.2.0
+  hooks:
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-shebang-scripts-are-executable
+  - id: check-toml
+  - id: check-yaml
+  - id: double-quote-string-fixer
+  - id: end-of-file-fixer
+  - id: mixed-line-ending
+    args: ['--fix=lf']
+  - id: requirements-txt-fixer
+  - id: trailing-whitespace
+- repo: https://github.com/myint/docformatter
+  rev: v1.4
+  hooks:
+  - id: docformatter
+    args: ['--in-place']
+- repo: https://github.com/pycqa/isort
+  rev: 5.10.1
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v0.991
+  hooks:
+    - id: mypy
+      args: ['--ignore-missing-imports']
+- repo: https://github.com/google/yapf
+  rev: v0.32.0
+  hooks:
+  - id: yapf
+    args: ['--parallel', '--in-place']

.style.yapf

@@ -0,0 +1,5 @@
+[style]
+based_on_style = pep8
+blank_line_before_nested_class_or_def = false
+spaces_before_comment = 2
+split_before_logical_operator = true

CutLER

@@ -0,0 +1 @@
+Subproject commit 077938c626341723050a1971107af552a6ca6697

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 hysts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE.CutLER

@@ -0,0 +1,202 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Don't include
+the brackets!)  The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -1,6 +1,6 @@
 ---
 title: MaskCut
-emoji: 💩
+emoji: 🌍
 colorFrom: pink
 colorTo: pink
 sdk: gradio

app.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env
+
+import pathlib
+
+import gradio as gr
+import numpy as np
+import PIL.Image as Image
+
+from model import Model, random_color, vis_mask
+
+model = Model()
+
+
+def run(image_path, threshold, max_num_mask):
+    image = np.asarray(Image.open(image_path).convert('RGB'))
+    masks = model(image_path, threshold, max_num_mask)
+    for mask in masks:
+        image = vis_mask(image, mask, random_color(rgb=True))
+    return image
+
+
+TITLE = 'MaskCut'
+DESCRIPTION = 'This is an unofficial demo for https://github.com/facebookresearch/CutLER.'
+
+paths = sorted(pathlib.Path('CutLER/maskcut/imgs').glob('*.jpg'))
+demo = gr.Interface(
+    fn=run,
+    inputs=[
+        gr.Image(label='Input image', type='filepath'),
+        gr.Slider(label='Threshold used for producing binary graph',
+                  minimum=0,
+                  maximum=1,
+                  value=0.15,
+                  step=0.01),
+        gr.Slider(label='The maximum number of pseudo-masks per image',
+                  minimum=1,
+                  maximum=20,
+                  value=3,
+                  step=1),
+    ],
+    outputs=gr.Image(label='Result', type='numpy'),
+    examples=[[path.as_posix(), 0.15, 3] for path in paths],
+    title=TITLE,
+    description=DESCRIPTION)
+demo.queue().launch()

model.py

@@ -0,0 +1,83 @@
+# This file is adapted from https://github.com/facebookresearch/CutLER/blob/077938c626341723050a1971107af552a6ca6697/maskcut/demo.py
+# The original license file is the file named LICENSE.CutLER in this repo.
+
+import sys
+
+import numpy as np
+import PIL.Image as Image
+import torch
+from scipy import ndimage
+
+sys.path.append('CutLER/maskcut/')
+sys.path.append('CutLER/')
+import dino
+from colormap import random_color
+from crf import densecrf
+from maskcut import maskcut
+from third_party.TokenCut.unsupervised_saliency_detection import metric
+
+
+def vis_mask(input, mask, mask_color):
+    fg = mask > 0.5
+    rgb = np.copy(input)
+    rgb[fg] = (rgb[fg] * 0.3 + np.array(mask_color) * 0.7).astype(np.uint8)
+    return Image.fromarray(rgb)
+
+
+class Model:
+    def __init__(self):
+        self.device = torch.device(
+            'cuda:0' if torch.cuda.is_available() else 'cpu')
+        self.backbone = self.load_backbone()
+
+    def load_backbone(self):
+        # DINO hyperparameters
+        vit_arch = 'base'
+        vit_feat = 'k'
+        patch_size = 8
+        # DINO pre-trained model
+        url = 'https://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pth'
+        feat_dim = 768
+
+        # extract patch features with a pretrained DINO model
+        backbone = dino.ViTFeat(url, feat_dim, vit_arch, vit_feat, patch_size)
+        backbone.eval()
+        backbone.to(self.device)
+        return backbone
+
+    def __call__(self, img_path, tau, n, fixed_size=480):
+        # get pseudo-masks with MaskCut
+        bipartitions, _, I_new = maskcut(img_path,
+                                         self.backbone,
+                                         self.backbone.patch_size,
+                                         tau,
+                                         N=n,
+                                         fixed_size=fixed_size,
+                                         cpu=self.device.type == 'cpu')
+        I = Image.open(img_path).convert('RGB')
+        width, height = I.size
+        pseudo_mask_list = []
+        for idx, bipartition in enumerate(bipartitions):
+            # post-process pseudo-masks with CRF
+            pseudo_mask = densecrf(np.array(I_new), bipartition)
+            pseudo_mask = ndimage.binary_fill_holes(pseudo_mask >= 0.5)
+
+            # filter out the mask that have a very different pseudo-mask after the CRF
+            mask1 = torch.from_numpy(bipartition).to(self.device)
+            mask2 = torch.from_numpy(pseudo_mask).to(self.device)
+            if metric.IoU(mask1, mask2) < 0.5:
+                pseudo_mask = pseudo_mask * -1
+
+            # construct binary pseudo-masks
+            pseudo_mask[pseudo_mask < 0] = 0
+            pseudo_mask = Image.fromarray(np.uint8(pseudo_mask * 255))
+            pseudo_mask = np.asarray(pseudo_mask.resize((width, height)))
+
+            pseudo_mask = pseudo_mask.astype(np.uint8)
+            upper = np.max(pseudo_mask)
+            lower = np.min(pseudo_mask)
+            thresh = upper / 2.0
+            pseudo_mask[pseudo_mask > thresh] = upper
+            pseudo_mask[pseudo_mask <= thresh] = lower
+            pseudo_mask_list.append(pseudo_mask)
+        return pseudo_mask_list

requirements.txt

@@ -0,0 +1,9 @@
+git+https://github.com/lucasb-eyer/pydensecrf@0d53acb
+gradio==3.16.2
+numpy==1.23.5
+opencv-python==4.6.0.66
+pycocotools==2.0.6
+scikit-image==0.19.2
+torch==1.13.1
+torchvision==0.14.1
+tqdm==4.64.1