all

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+SAM_counting_anything__ArXiv_.pdf filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Vision-Intelligence-and-Robots-Group
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,13 +1,54 @@
----
-title: Counting Anything
-emoji: 🐠
-colorFrom: gray
-colorTo: indigo
-sdk: gradio
-sdk_version: 3.27.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# count-anything
+An empirical study on few-shot counting using segment anything (SAM)
+
+Meta AI recently released the Segment Anything model [[SAM]](https://github.com/facebookresearch/segment-anything), which has garnered attention due to its impressive performance in class-agnostic segmenting. In this study, we explore the use of SAM for the challenging task of few-shot object counting, which involves counting objects of an unseen category by providing a few bounding boxes of examples. We compare SAM's performance with other few-shot counting methods and find that it is currently unsatisfactory without further fine-tuning, particularly for small and crowded objects.
+
+![image](example.png)
+## Install
+Install python dependencies. We use conda and python 3.10.4 and PyTorch 1.13.1
+> conda env create -f env.yaml
+
+## Dataset preparation
+- For FSC-147:
+    Images can be downloaded from here: https://drive.google.com/file/d/1ymDYrGs9DSRicfZbSCDiOu0ikGDh5k6S/view?usp=sharing
+
+- For coco val2017:
+    Images can be downloaded from here: https://cocodataset.org/
+## Comparison Results
+
+### FSC
+
+![image](resultFSC.png)
+
+### COCO
+
+![image](resultcoco.png)
+## Test
+Download the [ViT-H SAM model](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth)
+
+- For FSC-147:
+```
+python test_FSC.py --data_path <FSC-147 dataset path> --model_path <path to ViT-H SAM model>
+```
+
+- For coco val2017:
+```
+python test_coco.py --data_path <coco val2017 dataset path\> --model_path <path to ViT-H SAM model>
+```
+
+## Visualize
+You can run [vis_FSC.ipynb](vis_FSC.ipynb) for FSC-147 or [vis_coco.ipynb](vis_coco.ipynb) for coco.
+
+## Acknowledgement
+We thank facebookresearch for their segment-anything model [[project]](https://github.com/facebookresearch/segment-anything), cvlab-stonybrook for their Learning To Count Everything [[project]](https://github.com/cvlab-stonybrook/LearningToCountEverything) and coco [[datasets]](https://cocodataset.org/).
+
+## Citation
+If you find the code useful, please cite:
+```
+@article{ma2023countanything,
+  title={CAN SAM COUNT ANYTHING? AN EMPIRICAL STUDY ON SAM COUNTING},
+  author={Ma, Zhiheng and Hong, Xiaopeng and Shangguan Qinnan},
+  journal={arXiv preprint arXiv:2304.xxxxx},
+  year={2023}
+}
+```

SAM_counting_anything__ArXiv_.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49bcb63df39d9ad072ae11393355e29757169608fa4540388698cf23a2c7110a
+size 6071349

app.py

@@ -0,0 +1,88 @@
+from PIL import Image, ImageDraw
+import cv2
+import gradio as gr
+import torch
+from segment_anything import sam_model_registry
+from automatic_mask_generator import SamAutomaticMaskGenerator
+
+device = 'cuda'
+sam = sam_model_registry['vit_h'](checkpoint='./sam_vit_h_4b8939.pth')
+sam.to(device=device)
+
+
+mask_generator = SamAutomaticMaskGenerator(
+                                model=sam,
+                                min_mask_region_area=25
+                                )
+
+def binarize(x):
+    return (x != 0).astype('uint8') * 255
+
+def draw_box(boxes=[], img=None):
+    if len(boxes) == 0 and img is None:
+        return None
+
+    if img is None:
+        img = Image.new('RGB', (512, 512), (255, 255, 255))
+    colors = ["red", "olive", "blue", "green", "orange", "brown", "cyan", "purple"]
+    draw = ImageDraw.Draw(img)
+    # print(boxes)
+    for bid, box in enumerate(boxes):
+        draw.rectangle([box[0], box[1], box[2], box[3]], outline=colors[bid % len(colors)], width=4)
+    return img
+
+
+def draw_pred_box(boxes=[], img=None):
+    if len(boxes) == 0 and img is None:
+        return None
+
+    if img is None:
+        img = Image.new('RGB', (512, 512), (255, 255, 255))
+    colors = "green"
+    draw = ImageDraw.Draw(img)
+    # print(boxes)
+    for bid, box in enumerate(boxes):
+        draw.rectangle([box[0], box[1], box[2], box[3]], outline=colors, width=4)
+    return img
+
+
+def debug(input_img):
+    mask = input_img["mask"]
+    mask = mask[..., 0]
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    boxes = []
+    for contour in contours:
+        y1, y2 = contour[:, 0, 1].min(), contour[:, 0, 1].max()
+        x1, x2 = contour[:, 0, 0].min(), contour[:, 0, 0].max()
+        boxes.append([x1, y1, x2, y2])
+    draw_image = draw_box(boxes, Image.fromarray(input_img["image"]))
+
+    masks = mask_generator.generate(input_img["image"], boxes)
+    pred_cnt = len(masks)
+    pred_bboxes = []
+    for i in masks:
+        x0, y0, w, h = i['bbox']
+        pred_bboxes.append([x0, y0, x0+w, y0+h])
+    pred_image = draw_pred_box(pred_bboxes, Image.fromarray(input_img["image"]))
+    return [draw_image, pred_image, "Count: {}".format(pred_cnt)]
+
+description = """<p style="text-align: center; font-weight: bold;">
+    <span style="font-size: 28px">Count Anything</span>
+    <br>
+    <span style="font-size: 18px" id="paper-info">
+        [<a href=" " target="_blank">Project Page</a>]
+        [<a href=" " target="_blank">Paper</a>]
+        [<a href="https://github.com/Vision-Intelligence-and-Robots-Group/count-anything" target="_blank">GitHub</a>]
+    </span>
+</p>
+"""
+
+run = gr.Interface(
+    debug,
+    gr.Image(shape=[512, 512], source="upload", tool="sketch").style(height=500, width=500),
+    [gr.Image(), gr.Image(), gr.Text()],
+    description = description
+)
+
+run.launch()

automatic_mask_generator.py

@@ -0,0 +1,496 @@
+import numpy as np
+import torch
+from torchvision.ops.boxes import batched_nms, box_area  # type: ignore
+
+from typing import Any, Dict, List, Optional, Tuple
+import torch.nn.functional as F
+from collections import defaultdict
+
+from segment_anything.modeling import Sam
+from segment_anything.predictor import SamPredictor
+from segment_anything.utils.amg import (
+    MaskData,
+    area_from_rle,
+    batch_iterator,
+    batched_mask_to_box,
+    box_xyxy_to_xywh,
+    build_all_layer_point_grids,
+    calculate_stability_score,
+    coco_encode_rle,
+    generate_crop_boxes,
+    is_box_near_crop_edge,
+    mask_to_rle_pytorch,
+    remove_small_regions,
+    rle_to_mask,
+    uncrop_boxes_xyxy,
+    uncrop_masks,
+    uncrop_points,
+)
+
+
+def uncrop_boxes_xyxy(boxes: torch.Tensor, crop_box: List[int]) -> torch.Tensor:
+    x0, y0, _, _ = crop_box
+    offset = torch.tensor([[x0, y0, x0, y0]], device=boxes.device)
+    # Check if boxes has a channel dimension
+    if len(boxes.shape) == 3:
+        offset = offset.unsqueeze(1)
+    return boxes + offset
+
+def pre_process_ref_box(ref_box, crop_box, layer_idx):
+    if layer_idx == 0:
+        return ref_box
+    else:
+        new_bbox = []
+        x0, y0, x1, y1 = crop_box
+        for ref in ref_box:
+            x0_r, y0_r, x1_r, y1_r = ref
+            area = (y1_r - y0_r) * (x1_r - x0_r)
+            x_0_new = max(x0, x0_r)
+            y_0_new = max(y0, y0_r)
+            x_1_new = min(x1, x1_r)
+            y_1_new = min(y1, y1_r)
+            crop_area = (y_1_new - y_0_new) * (x_1_new - x_0_new)
+            if crop_area / area > 0.7:
+                new_bbox.append([x_0_new, y_0_new, x_1_new, y_1_new])
+
+        return new_bbox
+
+
+
+
+class SamAutomaticMaskGenerator:
+    def __init__(
+        self,
+        model: Sam,
+        points_per_side: Optional[int] = 32,
+        points_per_batch: int = 64,
+        pred_iou_thresh: float = 0.88,
+        stability_score_thresh: float = 0.95,
+        stability_score_offset: float = 1.0,
+        box_nms_thresh: float = 0.7,
+        crop_n_layers: int = 0,
+        crop_nms_thresh: float = 0.7,
+        crop_overlap_ratio: float = 512 / 1500,
+        crop_n_points_downscale_factor: int = 1,
+        point_grids: Optional[List[np.ndarray]] = None,
+        min_mask_region_area: int = 0,
+        output_mode: str = "binary_mask",
+    ) -> None:
+        """
+        Using a SAM model, generates masks for the entire image.
+        Generates a grid of point prompts over the image, then filters
+        low quality and duplicate masks. The default settings are chosen
+        for SAM with a ViT-H backbone.
+
+        Arguments:
+          model (Sam): The SAM model to use for mask prediction.
+          points_per_side (int or None): The number of points to be sampled
+            along one side of the image. The total number of points is
+            points_per_side**2. If None, 'point_grids' must provide explicit
+            point sampling.
+          points_per_batch (int): Sets the number of points run simultaneously
+            by the model. Higher numbers may be faster but use more GPU memory.
+          pred_iou_thresh (float): A filtering threshold in [0,1], using the
+            model's predicted mask quality.
+          stability_score_thresh (float): A filtering threshold in [0,1], using
+            the stability of the mask under changes to the cutoff used to binarize
+            the model's mask predictions.
+          stability_score_offset (float): The amount to shift the cutoff when
+            calculated the stability score.
+          box_nms_thresh (float): The box IoU cutoff used by non-maximal
+            suppression to filter duplicate masks.
+          crops_n_layers (int): If >0, mask prediction will be run again on
+            crops of the image. Sets the number of layers to run, where each
+            layer has 2**i_layer number of image crops.
+          crops_nms_thresh (float): The box IoU cutoff used by non-maximal
+            suppression to filter duplicate masks between different crops.
+          crop_overlap_ratio (float): Sets the degree to which crops overlap.
+            In the first crop layer, crops will overlap by this fraction of
+            the image length. Later layers with more crops scale down this overlap.
+          crop_n_points_downscale_factor (int): The number of points-per-side
+            sampled in layer n is scaled down by crop_n_points_downscale_factor**n.
+          point_grids (list(np.ndarray) or None): A list over explicit grids
+            of points used for sampling, normalized to [0,1]. The nth grid in the
+            list is used in the nth crop layer. Exclusive with points_per_side.
+          min_mask_region_area (int): If >0, postprocessing will be applied
+            to remove disconnected regions and holes in masks with area smaller
+            than min_mask_region_area. Requires opencv.
+          output_mode (str): The form masks are returned in. Can be 'binary_mask',
+            'uncompressed_rle', or 'coco_rle'. 'coco_rle' requires pycocotools.
+            For large resolutions, 'binary_mask' may consume large amounts of
+            memory.
+        """
+
+        assert (points_per_side is None) != (
+            point_grids is None
+        ), "Exactly one of points_per_side or point_grid must be provided."
+        if points_per_side is not None:
+            self.point_grids = build_all_layer_point_grids(
+                points_per_side,
+                crop_n_layers,
+                crop_n_points_downscale_factor,
+            )
+        elif point_grids is not None:
+            self.point_grids = point_grids
+        else:
+            raise ValueError("Can't have both points_per_side and point_grid be None.")
+
+        assert output_mode in [
+            "binary_mask",
+            "uncompressed_rle",
+            "coco_rle",
+        ], f"Unknown output_mode {output_mode}."
+        if output_mode == "coco_rle":
+            from pycocotools import mask as mask_utils  # type: ignore # noqa: F401
+
+        if min_mask_region_area > 0:
+            import cv2  # type: ignore # noqa: F401
+
+        self.predictor = SamPredictor(model)
+        self.points_per_batch = points_per_batch
+        self.pred_iou_thresh = pred_iou_thresh
+        self.stability_score_thresh = stability_score_thresh
+        self.stability_score_offset = stability_score_offset
+        self.box_nms_thresh = box_nms_thresh
+        self.crop_n_layers = crop_n_layers
+        self.crop_nms_thresh = crop_nms_thresh
+        self.crop_overlap_ratio = crop_overlap_ratio
+        self.crop_n_points_downscale_factor = crop_n_points_downscale_factor
+        self.min_mask_region_area = min_mask_region_area
+        self.output_mode = output_mode
+
+        self.prototype = defaultdict(list)
+
+    @torch.no_grad()
+    def generate(self, image: np.ndarray, ref_bbox) -> List[Dict[str, Any]]:
+        """
+        Generates masks for the given image.
+
+        Arguments:
+          image (np.ndarray): The image to generate masks for, in HWC uint8 format.
+
+
+        Returns:
+           list(dict(str, any)): A list over records for masks. Each record is
+             a dict containing the following keys:
+               segmentation (dict(str, any) or np.ndarray): The mask. If
+                 output_mode='binary_mask', is an array of shape HW. Otherwise,
+                 is a dictionary containing the RLE.
+               bbox (list(float)): The box around the mask, in XYWH format.
+               area (int): The area in pixels of the mask.
+               predicted_iou (float): The model's own prediction of the mask's
+                 quality. This is filtered by the pred_iou_thresh parameter.
+               point_coords (list(list(float))): The point coordinates input
+                 to the model to generate this mask.
+               stability_score (float): A measure of the mask's quality. This
+                 is filtered on using the stability_score_thresh parameter.
+               crop_box (list(float)): The crop of the image used to generate
+                 the mask, given in XYWH format.
+        """
+
+        # Generate masks
+        mask_data = self._generate_masks(image, ref_bbox)
+
+        # Filter small disconnected regions and holes in masks
+        if self.min_mask_region_area > 0:
+            mask_data = self.postprocess_small_regions(
+                mask_data,
+                self.min_mask_region_area,
+                max(self.box_nms_thresh, self.crop_nms_thresh),
+            )
+
+        # Encode masks
+        if self.output_mode == "coco_rle":
+            mask_data["segmentations"] = [coco_encode_rle(rle) for rle in mask_data["rles"]]
+        elif self.output_mode == "binary_mask":
+            mask_data["segmentations"] = [rle_to_mask(rle) for rle in mask_data["rles"]]
+        else:
+            mask_data["segmentations"] = mask_data["rles"]
+
+        # Write mask records
+        curr_anns = []
+        for idx in range(len(mask_data["segmentations"])):
+            ann = {
+                "segmentation": mask_data["segmentations"][idx],
+                "area": area_from_rle(mask_data["rles"][idx]),
+                "bbox": box_xyxy_to_xywh(mask_data["boxes"][idx]).tolist(),
+                "predicted_iou": mask_data["iou_preds"][idx].item(),
+                "point_coords": [mask_data["points"][idx].tolist()],
+                "stability_score": mask_data["stability_score"][idx].item(),
+                "crop_box": box_xyxy_to_xywh(mask_data["crop_boxes"][idx]).tolist(),
+            }
+            curr_anns.append(ann)
+
+        return curr_anns
+
+    def _generate_masks(self, image: np.ndarray, ref_box) -> MaskData:
+        orig_size = image.shape[:2]
+        crop_boxes, layer_idxs = generate_crop_boxes(
+            orig_size, self.crop_n_layers, self.crop_overlap_ratio
+        )
+
+        # Iterate over image crops
+        # data = MaskData()
+        data_dic = defaultdict(MaskData)
+        for crop_box, layer_idx in zip(crop_boxes, layer_idxs):
+            crop_data = self._process_crop(image, crop_box, layer_idx, orig_size, ref_box)
+            data_dic[layer_idx].cat(crop_data)
+
+        data = MaskData()
+        for layer_idx in data_dic.keys():
+            proto_fea = torch.concat(self.prototype[layer_idx], dim=0)
+            if len(proto_fea) > 1:
+                cos_dis = proto_fea @ proto_fea.t()
+                sim_thresh = torch.min(cos_dis)
+            else:
+                sim_thresh = 0.7
+            sub_data = data_dic[layer_idx]
+            fea = sub_data['fea']
+            cos_dis = torch.max(fea @ proto_fea.t(), dim=1)[0]
+            sub_data.filter(cos_dis>=sim_thresh)
+            data.cat(sub_data)
+
+        self.prototype = defaultdict(list)
+
+
+        # Remove duplicate masks between crops
+        if len(crop_boxes) > 1:
+            # Prefer masks from smaller crops
+            scores = 1 / box_area(data["crop_boxes"])
+            scores = scores.to(data["boxes"].device)
+            keep_by_nms = batched_nms(
+                data["boxes"].float(),
+                scores,
+                torch.zeros(len(data["boxes"])),  # categories
+                iou_threshold=self.crop_nms_thresh,
+            )
+            data.filter(keep_by_nms)
+
+        data.to_numpy()
+        return data
+
+    def _process_crop(
+        self,
+        image: np.ndarray,
+        crop_box: List[int],
+        crop_layer_idx: int,
+        orig_size: Tuple[int, ...],
+        ref_box
+    ) -> MaskData:
+        # Crop the image and calculate embeddings
+        x0, y0, x1, y1 = crop_box
+        cropped_im = image[y0:y1, x0:x1, :]
+        cropped_im_size = cropped_im.shape[:2]
+        self.predictor.set_image(cropped_im)
+
+        ref_box = pre_process_ref_box(ref_box, crop_box, crop_layer_idx)
+        if len(ref_box) > 0:
+            ref_box = torch.tensor(ref_box, device=self.predictor.device)
+            transformed_boxes = self.predictor.transform.apply_boxes_torch(ref_box, cropped_im_size)
+            masks, iou_preds, low_res_masks = self.predictor.predict_torch(
+                point_coords=None,
+                point_labels=None,
+                boxes=transformed_boxes,
+                multimask_output=False
+                )
+            feature = self.predictor.get_image_embedding()
+
+            low_res_masks = F.interpolate(low_res_masks, size=feature.shape[-2:], mode='bilinear', align_corners=False)
+            
+            feature = feature.flatten(2, 3)
+            low_res_masks = low_res_masks.flatten(2, 3)
+            masks_low_res = (low_res_masks > self.predictor.model.mask_threshold).float()
+            topk_idx = torch.topk(low_res_masks, 1)[1]
+            masks_low_res.scatter_(2, topk_idx, 1.0)
+
+
+            prototype_fea = (feature * masks_low_res).sum(dim=2) / masks_low_res.sum(dim=2)
+            prototype_fea = F.normalize(prototype_fea, dim=1)
+            self.prototype[crop_layer_idx].append(prototype_fea)
+
+
+        if crop_layer_idx == 0:                                     # add reference gounding
+            x = ref_box[:, 0] + (ref_box[:, 2] - ref_box[:, 0]) / 2
+            y = ref_box[:, 1] + (ref_box[:, 3] - ref_box[:, 1]) / 2
+            points = torch.stack([x, y], dim=1)
+            data = MaskData(
+                masks=masks.flatten(0, 1),
+                iou_preds= torch.ones_like(iou_preds.flatten(0, 1)),
+                fea = prototype_fea,
+                points=points.cpu(),
+                stability_score = torch.ones_like(iou_preds.flatten(0, 1)),
+            )
+            data["boxes"] = batched_mask_to_box(data["masks"])
+            data["rles"] = mask_to_rle_pytorch(data["masks"])
+            del data["masks"]
+        else:
+            data = MaskData()
+
+        
+
+        # Get points for this crop
+        points_scale = np.array(cropped_im_size)[None, ::-1]
+        points_for_image = self.point_grids[crop_layer_idx] * points_scale
+
+        # Generate masks for this crop in batches
+        for (points,) in batch_iterator(self.points_per_batch, points_for_image):
+            batch_data = self._process_batch(points, cropped_im_size, 
+                                             crop_box, orig_size)
+            data.cat(batch_data)
+            del batch_data
+        self.predictor.reset_image()
+
+        # Remove duplicates within this crop.
+        keep_by_nms = batched_nms(
+            data["boxes"].float(),
+            data["iou_preds"],
+            torch.zeros(len(data["boxes"])),  # categories
+            iou_threshold=self.box_nms_thresh,
+        )
+        data.filter(keep_by_nms)
+
+        # Return to the original image frame
+        data["boxes"] = uncrop_boxes_xyxy(data["boxes"], crop_box)
+        data["points"] = uncrop_points(data["points"], crop_box)
+        data["crop_boxes"] = torch.tensor([crop_box for _ in range(len(data["rles"]))])
+
+        return data
+
+    def _process_batch(
+        self,
+        points: np.ndarray,
+        im_size: Tuple[int, ...],
+        crop_box: List[int],
+        orig_size: Tuple[int, ...],
+    ) -> MaskData:
+        orig_h, orig_w = orig_size
+
+        # Run model on this batch
+        transformed_points = self.predictor.transform.apply_coords(points, im_size)
+        in_points = torch.as_tensor(transformed_points, device=self.predictor.device)
+        in_labels = torch.ones(in_points.shape[0], dtype=torch.int, device=in_points.device)
+        masks, iou_preds, low_res_masks = self.predictor.predict_torch(
+            in_points[:, None, :],
+            in_labels[:, None],
+            multimask_output=True,
+            return_logits=True,
+        )
+
+        feature = self.predictor.get_image_embedding()
+        low_res_masks=low_res_masks.flatten(0, 1)
+        low_res_masks = F.interpolate(low_res_masks[:, None, :, :], size=feature.shape[-2:], 
+                                      mode='bilinear', align_corners=False)
+        # low_res_masks = low_res_masks > self.predictor.model.mask_threshold
+
+        # fea = feature.flatten(2, 3)
+        # low_res_masks = low_res_masks.flatten(2, 3)
+        # topk_idx = torch.topk(low_res_masks, 4)[1]
+        # fea = fea.expand(topk_idx.shape[0], -1, -1)
+        # topk_idx = topk_idx.expand(-1, fea.shape[1], -1)
+        # fea = fea.gather(2, topk_idx)
+
+
+        feature = feature.flatten(2, 3)
+        low_res_masks = low_res_masks.flatten(2, 3)
+        masks_low_res = (low_res_masks > self.predictor.model.mask_threshold).float()
+        topk_idx = torch.topk(low_res_masks, 1)[1]
+        masks_low_res.scatter_(2, topk_idx, 1.0)
+        pool_fea = (feature * masks_low_res).sum(dim=2) / masks_low_res.sum(dim=2)
+        pool_fea = F.normalize(pool_fea, dim=1)
+
+        # k_val = torch.topk(torch.flatten(low_res_masks, start_dim=2, end_dim=3), k=4, dim=-1)[0][:, :, -1, None]
+        # low_res_masks = (low_res_masks >= k_val).float()
+        # low_res_masks = low_res_masks.float()
+        # pool_fea = (feature * low_res_masks).sum(dim=(2, 3)) / low_res_masks.sum(dim=(2, 3))
+
+
+
+        # Serialize predictions and store in MaskData
+        data = MaskData(
+            masks=masks.flatten(0, 1),
+            iou_preds=iou_preds.flatten(0, 1),
+            points=torch.as_tensor(points.repeat(masks.shape[1], axis=0)),
+            fea = pool_fea,
+        )
+        del masks
+
+
+        # Filter by predicted IoU
+        if self.pred_iou_thresh > 0.0:
+            keep_mask = data["iou_preds"] > self.pred_iou_thresh
+            data.filter(keep_mask)
+
+        # Calculate stability score
+        data["stability_score"] = calculate_stability_score(
+            data["masks"], self.predictor.model.mask_threshold, self.stability_score_offset
+        )
+        if self.stability_score_thresh > 0.0:
+            keep_mask = data["stability_score"] >= self.stability_score_thresh
+            data.filter(keep_mask)
+
+        # Threshold masks and calculate boxes
+        data["masks"] = data["masks"] > self.predictor.model.mask_threshold
+        data["boxes"] = batched_mask_to_box(data["masks"])
+
+        # Filter boxes that touch crop boundaries
+        keep_mask = ~is_box_near_crop_edge(data["boxes"], crop_box, [0, 0, orig_w, orig_h])
+        if not torch.all(keep_mask):
+            data.filter(keep_mask)
+
+        # Compress to RLE
+        data["masks"] = uncrop_masks(data["masks"], crop_box, orig_h, orig_w)
+        data["rles"] = mask_to_rle_pytorch(data["masks"])
+        del data["masks"]
+
+        return data
+
+    @staticmethod
+    def postprocess_small_regions(
+        mask_data: MaskData, min_area: int, nms_thresh: float
+    ) -> MaskData:
+        """
+        Removes small disconnected regions and holes in masks, then reruns
+        box NMS to remove any new duplicates.
+
+        Edits mask_data in place.
+
+        Requires open-cv as a dependency.
+        """
+        if len(mask_data["rles"]) == 0:
+            return mask_data
+
+        # Filter small disconnected regions and holes
+        new_masks = []
+        scores = []
+        for rle in mask_data["rles"]:
+            mask = rle_to_mask(rle)
+
+            mask, changed = remove_small_regions(mask, min_area, mode="holes")
+            unchanged = not changed
+            mask, changed = remove_small_regions(mask, min_area, mode="islands")
+            unchanged = unchanged and not changed
+
+            new_masks.append(torch.as_tensor(mask).unsqueeze(0))
+            # Give score=0 to changed masks and score=1 to unchanged masks
+            # so NMS will prefer ones that didn't need postprocessing
+            scores.append(float(unchanged))
+
+        # Recalculate boxes and remove any new duplicates
+        masks = torch.cat(new_masks, dim=0)
+        boxes = batched_mask_to_box(masks)
+        keep_by_nms = batched_nms(
+            boxes.float(),
+            torch.as_tensor(scores),
+            torch.zeros(len(boxes)),  # categories
+            iou_threshold=nms_thresh,
+        )
+
+        # Only recalculate RLEs for masks that have changed
+        for i_mask in keep_by_nms:
+            if scores[i_mask] == 0.0:
+                mask_torch = masks[i_mask].unsqueeze(0)
+                mask_data["rles"][i_mask] = mask_to_rle_pytorch(mask_torch)[0]
+                mask_data["boxes"][i_mask] = boxes[i_mask]  # update res directly
+        mask_data.filter(keep_by_nms)
+
+        return mask_data

env.yaml

@@ -0,0 +1,177 @@
+name: ltce
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
+  - blas=1.0=openblas
+  - brotli=1.0.9=h5eee18b_7
+  - brotli-bin=1.0.9=h5eee18b_7
+  - bzip2=1.0.8=h7b6447c_0
+  - ca-certificates=2023.01.10=h06a4308_0
+  - cairo=1.16.0=hb05425b_4
+  - certifi=2022.12.7=py310h06a4308_0
+  - contourpy=1.0.5=py310hdb19cb5_0
+  - cycler=0.11.0=pyhd3eb1b0_0
+  - dbus=1.13.18=hb2f20db_0
+  - debugpy=1.5.1=py310h295c915_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - eigen=3.4.0=h4bd325d_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - expat=2.2.10=h9c3ff4c_0
+  - ffmpeg=4.2.2=h20bf706_0
+  - fontconfig=2.14.1=hef1e5e3_0
+  - fonttools=4.25.0=pyhd3eb1b0_0
+  - freetype=2.10.4=h0708190_1
+  - giflib=5.2.1=h36c2ea0_2
+  - glib=2.69.1=h4ff587b_1
+  - gmp=6.2.1=h58526e2_0
+  - gnutls=3.6.13=h85f3911_1
+  - graphite2=1.3.14=h295c915_1
+  - gst-plugins-base=1.14.1=h6a678d5_1
+  - gstreamer=1.14.1=h5eee18b_1
+  - harfbuzz=4.3.0=hf52aaf7_1
+  - hdf5=1.10.6=h3ffc7dd_1
+  - icu=58.2=hf484d3e_1000
+  - ipykernel=6.15.0=pyh210e3f2_0
+  - ipython=8.12.0=pyh41d4057_0
+  - jedi=0.18.2=pyhd8ed1ab_0
+  - jpeg=9e=h166bdaf_1
+  - jupyter_client=7.3.4=pyhd8ed1ab_0
+  - jupyter_core=5.3.0=py310hff52083_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.4=py310h6a678d5_0
+  - krb5=1.19.3=h3790be6_0
+  - lame=3.100=h7f98852_1001
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - libblas=3.9.0=15_linux64_openblas
+  - libbrotlicommon=1.0.9=h5eee18b_7
+  - libbrotlidec=1.0.9=h5eee18b_7
+  - libbrotlienc=1.0.9=h5eee18b_7
+  - libcblas=3.9.0=15_linux64_openblas
+  - libclang=10.0.1=default_hb85057a_2
+  - libdeflate=1.17=h5eee18b_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libevent=2.1.12=h8f2d780_0
+  - libffi=3.3=he6710b0_2
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgfortran-ng=12.2.0=h69a702a_19
+  - libgfortran5=12.2.0=h337968e_19
+  - libgomp=11.2.0=h1234567_1
+  - liblapack=3.9.0=15_linux64_openblas
+  - libllvm10=10.0.1=he513fc3_3
+  - libopenblas=0.3.20=pthreads_h78a6416_0
+  - libopus=1.3.1=h7f98852_1
+  - libpng=1.6.39=h5eee18b_0
+  - libpq=12.9=h16c4e8d_3
+  - libprotobuf=3.20.3=he621ea3_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - libtiff=4.5.0=h6a678d5_2
+  - libuuid=1.41.5=h5eee18b_0
+  - libvpx=1.7.0=h439df22_0
+  - libwebp=1.2.4=h11a3e52_1
+  - libwebp-base=1.2.4=h5eee18b_1
+  - libxcb=1.15=h7f8727e_0
+  - libxkbcommon=1.0.1=hfa300c1_0
+  - libxml2=2.9.14=h74e7548_0
+  - libxslt=1.1.35=h4e12654_0
+  - lz4-c=1.9.3=h9c3ff4c_1
+  - matplotlib=3.7.1=py310h06a4308_1
+  - matplotlib-base=3.7.1=py310h1128e8f_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - munkres=1.1.4=py_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - nettle=3.6=he412f7d_0
+  - nspr=4.33=h295c915_0
+  - nss=3.74=h0370c37_0
+  - opencv=4.6.0=py310h1128e8f_3
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=1.1.1t=h7f8727e_0
+  - packaging=23.1=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pcre=8.45=h9c3ff4c_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.0.1=py310h06a4308_0
+  - pixman=0.40.0=h36c2ea0_0
+  - platformdirs=3.2.0=pyhd8ed1ab_0
+  - ply=3.11=py310h06a4308_0
+  - prompt-toolkit=3.0.38=pyha770c72_0
+  - prompt_toolkit=3.0.38=hd8ed1ab_0
+  - psutil=5.9.0=py310h5eee18b_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.15.0=pyhd8ed1ab_0
+  - pyparsing=3.0.9=py310h06a4308_0
+  - pyqt=5.15.7=py310h6a678d5_1
+  - python=3.10.4=h12debd9_0
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.10=2_cp310
+  - pyzmq=23.2.0=py310h6a678d5_0
+  - qt-main=5.15.2=h327a75a_7
+  - qt-webengine=5.15.9=hd2b0992_4
+  - qtwebkit=5.212=h4eab89a_4
+  - readline=8.2=h5eee18b_0
+  - setuptools=65.6.3=py310h06a4308_0
+  - sip=6.6.2=py310h6a678d5_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h1ccaba5_0
+  - toml=0.10.2=pyhd3eb1b0_0
+  - tornado=6.1=py310h5764c6d_3
+  - tqdm=4.65.0=py310h2f386ee_0
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - tzdata=2023c=h04d1e81_0
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - wheel=0.38.4=py310h06a4308_0
+  - x264=1!157.20191217=h7b6447c_0
+  - xz=5.2.10=h5eee18b_1
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.2=ha4553b6_0
+  - pip:
+    - charset-normalizer==3.1.0
+    - cmake==3.26.3
+    - filelock==3.11.0
+    - idna==3.4
+    - jinja2==3.1.2
+    - lit==16.0.1
+    - markupsafe==2.1.2
+    - mpmath==1.3.0
+    - networkx==3.1
+    - numpy==1.24.2
+    - nvidia-cublas-cu11==11.10.3.66
+    - nvidia-cuda-cupti-cu11==11.7.101
+    - nvidia-cuda-nvrtc-cu11==11.7.99
+    - nvidia-cuda-runtime-cu11==11.7.99
+    - nvidia-cudnn-cu11==8.5.0.96
+    - nvidia-cufft-cu11==10.9.0.58
+    - nvidia-curand-cu11==10.2.10.91
+    - nvidia-cusolver-cu11==11.4.0.1
+    - nvidia-cusparse-cu11==11.7.4.91
+    - nvidia-nccl-cu11==2.14.3
+    - nvidia-nvtx-cu11==11.7.91
+    - pillow==9.5.0
+    - pyqt5-sip==12.11.0
+    - requests==2.28.2
+    - segment-anything==1.0
+    - sympy==1.11.1
+    - torch==2.0.0
+    - torchaudio==2.0.1
+    - torchvision==0.15.1
+    - triton==2.0.0
+    - urllib3==1.26.15

test_FSC.py

@@ -0,0 +1,120 @@
+import cv2
+import argparse
+import json
+import numpy as np
+from tqdm import tqdm
+from os.path import exists
+import os
+
+from segment_anything import sam_model_registry
+from automatic_mask_generator import SamAutomaticMaskGenerator
+import matplotlib.pyplot as plt
+
+
+
+
+parser = argparse.ArgumentParser(description="Few Shot Counting Evaluation code")
+parser.add_argument("-dp", "--data_path", type=str, default='/data/counte/', help="Path to the FSC147 dataset")
+parser.add_argument("-ts", "--test_split", type=str, default='val', choices=["val_PartA","val_PartB","test_PartA","test_PartB","test", "val"], help="what data split to evaluate on")
+parser.add_argument("-mt", "--model_type", type=str, default="vit_h", help="model type")
+parser.add_argument("-mp",  "--model_path", type=str, default="/home/teddy/segment-anything/sam_vit_h_4b8939.pth", help="path to trained model")
+parser.add_argument("-v",  "--viz", type=bool, default=True, help="wether to visualize")
+parser.add_argument("-d",   "--device", default='0', help='assign device')
+args = parser.parse_args()
+
+data_path = args.data_path
+anno_file = data_path + 'annotation_FSC147_384.json'
+data_split_file = data_path + 'Train_Test_Val_FSC_147.json'
+im_dir = data_path + 'images_384_VarV2'
+
+
+if not exists(anno_file) or not exists(im_dir):
+    print("Make sure you set up the --data-path correctly.")
+    print("Current setting is {}, but the image dir and annotation file do not exist.".format(args.data_path))
+    print("Aborting the evaluation")
+    exit(-1)
+
+def show_anns(anns):
+    if len(anns) == 0:
+        return
+    sorted_anns = sorted(anns, key=(lambda x: x['area']), reverse=True)
+    ax = plt.gca()
+    ax.set_autoscale_on(False)
+    for ann in sorted_anns:
+        x0, y0, w, h = ann['bbox']
+        ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2))
+        ax.scatter([x0+w//2], [y0+h//2], color='green', marker='*', s=10, edgecolor='white', linewidth=1.25)
+
+
+debug = True
+os.environ['CUDA_VISIBLE_DEVICES'] = args.device.strip()
+device = 'cuda'
+sam = sam_model_registry[args.model_type](checkpoint=args.model_path)
+sam.to(device=device)
+
+
+mask_generator = SamAutomaticMaskGenerator(
+                                model=sam,
+                                min_mask_region_area=25
+                                )
+
+with open(anno_file) as f:
+    annotations = json.load(f)
+
+with open(data_split_file) as f:
+    data_split = json.load(f)
+
+
+cnt = 0
+SAE = 0  # sum of absolute errors
+SSE = 0  # sum of square errors
+
+print("Evaluation on {} data".format(args.test_split))
+im_ids = data_split[args.test_split]
+
+# with open("err.json") as f:
+#     im_ids = json.load(f)
+
+
+pbar = tqdm(im_ids)
+# err_list = []
+for im_id in pbar:
+    anno = annotations[im_id]
+    bboxes = anno['box_examples_coordinates']
+    dots = np.array(anno['points'])
+
+    image = cv2.imread('{}/{}'.format(im_dir, im_id))
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+    input_boxes = list()
+    for bbox in bboxes:
+        x1, y1 = bbox[0][0], bbox[0][1]
+        x2, y2 = bbox[2][0], bbox[2][1]
+        input_boxes.append([x1, y1, x2, y2])
+    
+    masks = mask_generator.generate(image, input_boxes)
+    if args.viz:
+        if not exists('viz'):
+            os.mkdir('viz')
+        plt.figure(figsize=(10,10))
+        plt.imshow(image)
+        show_anns(masks)
+        plt.axis('off')
+        plt.savefig('viz/{}'.format(im_id))
+        plt.close()
+
+    gt_cnt = dots.shape[0]
+    pred_cnt = len(masks)
+    cnt = cnt + 1
+    err = abs(gt_cnt - pred_cnt)
+    SAE += err
+    SSE += err**2
+    # if err / gt_cnt > 0.7:
+    #     err_list.append(im_id)
+
+    pbar.set_description('{:<8}: actual-predicted: {:6d}, {:6.1f}, error: {:6.1f}. Current MAE: {:5.2f}, RMSE: {:5.2f}'.\
+                         format(im_id, gt_cnt, pred_cnt, abs(pred_cnt - gt_cnt), SAE/cnt, (SSE/cnt)**0.5))
+
+print('On {} data, MAE: {:6.2f}, RMSE: {:6.2f}'.format(args.test_split, SAE/cnt, (SSE/cnt)**0.5))
+# with open('err.json', "w") as f:
+#     json.dump(err_list, f)

test_coco.py

@@ -0,0 +1,159 @@
+import cv2
+import argparse
+import json
+import numpy as np
+from tqdm import tqdm
+from os.path import exists
+import os
+
+from segment_anything import sam_model_registry
+from automatic_mask_generator import SamAutomaticMaskGenerator
+import matplotlib.pyplot as plt
+
+
+
+
+parser = argparse.ArgumentParser(description="Few Shot Counting Evaluation code")
+parser.add_argument("-dp", "--data_path", type=str, default='/data/counte/', help="Path to the coco dataset")
+parser.add_argument("-ts", "--test_split", type=str, default='val2017', choices=["val2017"], help="what data split to evaluate on")
+parser.add_argument("-mt", "--model_type", type=str, default="vit_h", help="model type")
+parser.add_argument("-mp",  "--model_path", type=str, default="/home/teddy/segment-anything/sam_vit_h_4b8939.pth", help="path to trained model")
+parser.add_argument("-v",  "--viz", type=bool, default=True, help="wether to visualize")
+parser.add_argument("-d",   "--device", default='0', help='assign device')
+args = parser.parse_args()
+
+data_path = args.data_path
+anno_file = data_path + 'annotations_trainval2017/annotations/instances_val2017.json'
+im_dir = data_path + 'val2017'
+
+
+if not exists(anno_file) or not exists(im_dir):
+    print("Make sure you set up the --data-path correctly.")
+    print("Current setting is {}, but the image dir and annotation file do not exist.".format(args.data_path))
+    print("Aborting the evaluation")
+    exit(-1)
+
+def show_anns(anns):
+    if len(anns) == 0:
+        return
+    sorted_anns = sorted(anns, key=(lambda x: x['area']), reverse=True)
+    ax = plt.gca()
+    ax.set_autoscale_on(False)
+    for ann in sorted_anns:
+        x0, y0, w, h = ann['bbox']
+        ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2))
+        ax.scatter([x0+w//2], [y0+h//2], color='green', marker='*', s=10, edgecolor='white', linewidth=1.25)
+
+
+debug = True
+os.environ['CUDA_VISIBLE_DEVICES'] = args.device.strip()
+device = 'cuda'
+sam = sam_model_registry[args.model_type](checkpoint=args.model_path)
+sam.to(device=device)
+
+
+mask_generator = SamAutomaticMaskGenerator(
+                                model=sam,
+                                min_mask_region_area=25
+                                )
+
+with open(anno_file) as f:
+    annotations = json.load(f)
+    
+images = sorted(annotations['images'],key=lambda x:x['file_name'])
+
+prepared_json = {}
+for i in images:
+    prepared_json[i['file_name']] = {
+        "H":i['height'],
+        "W":i['width'],
+        "boxes":{},
+        # "category_ids":[],
+    }
+for i in annotations['annotations']:
+    im_id = str(i['image_id'])
+    prezero = 12 - len(im_id)
+    im_id = '0'*prezero + im_id + ".jpg"
+    if i["category_id"] in prepared_json[im_id]["boxes"]:
+        prepared_json[im_id]["boxes"][i["category_id"]].append(i['bbox'])
+    else:
+        prepared_json[im_id]["boxes"][i["category_id"]] = []
+        prepared_json[im_id]["boxes"][i["category_id"]].append(i['bbox'])
+
+im_ids = []
+for i in prepared_json.keys():
+    im_ids.append(i)
+
+
+cnt = 0
+folds = [
+    [1,5,9,14,18,22,27,33,37,41,46,50,54,58,62,67,74,78,82,87],
+    [2,6,10,15,19,23,28,34,38,42,47,51,55,59,63,70,75,79,84,88],
+    [3,7,11,16,20,24,31,35,39,43,48,52,56,60,64,72,76,80,85,89],
+    [4,8,13,17,21,25,32,36,40,44,49,53,57,61,65,73,77,81,86,90],
+]
+SAE = [0,0,0,0]  # sum of absolute errors
+SSE = [0,0,0,0]  # sum of square errors
+
+print("Evaluation on {} data".format(args.test_split))
+
+# logs = []
+
+
+pbar = tqdm(im_ids)
+# err_list = []
+for im_id in pbar:
+    category_id = list(prepared_json[im_id]['boxes'].keys())
+    
+    image = cv2.imread('{}/{}'.format(im_dir, im_id))
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    # log = []
+    # log.append(im_id)
+    
+    for id in category_id:
+        boxes = prepared_json[im_id]['boxes'][id]
+        
+        input_boxes = list()
+        x1, y1 = boxes[0][0],boxes[0][1]
+        x2, y2 = boxes[0][0] + boxes[0][2],boxes[0][1] + boxes[0][3]
+        input_boxes.append([x1, y1, x2, y2])
+        
+        masks = mask_generator.generate(image, input_boxes)
+        
+        if args.viz:
+            if not exists('viz'):
+                os.mkdir('viz')
+            plt.figure(figsize=(10,10))
+            plt.imshow(image)
+            show_anns(masks)
+            plt.axis('off')
+            plt.savefig('viz/{}_{}.jpg'.format(im_id[0:-4],id))
+            plt.close()
+        
+        gt_cnt = len(boxes)
+        pred_cnt = len(masks)
+        err = abs(gt_cnt - pred_cnt)
+        log.append("\n{},gt_cnt:{},pred_cnt:{}".format(id,gt_cnt,pred_cnt))
+        if id in folds[0]:
+            SAE[0] += err
+            SSE[0] += err**2
+        elif id in folds[1]:
+            SAE[1] += err
+            SSE[1] += err**2
+        elif id in folds[2]:
+            SAE[2] += err
+            SSE[2] += err**2
+        elif id in folds[3]: 
+            SAE[3] += err
+            SSE[3] += err**2
+
+    cnt = cnt + 1
+    # logs.append(log)
+    pbar.set_description('fold1: {:6.2f}, fold2: {:6.2f}, fold3: {:6.2f}, fold4: {:6.2f},'.\
+                        format(SAE[0]/cnt,SAE[1]/cnt,SAE[2]/cnt,SAE[3]/cnt))
+    
+print('On {} data, fold1 MAE: {:6.2f}, RMSE: {:6.2f}\n \
+    fold2 MAE: {:6.2f}, RMSE: {:6.2f}\n \
+    fold3 MAE: {:6.2f}, RMSE: {:6.2f}\n \
+    fold4 MAE: {:6.2f}, RMSE: {:6.2f}\n \
+    '.format(args.test_split,SAE[0]/cnt,(SSE[0]/cnt)**0.5,SAE[1]/cnt,(SSE[1]/cnt)**0.5,SAE[2]/cnt,(SSE[2]/cnt)**0.5,SAE[3]/cnt,(SSE[3]/cnt)**0.5))