From 317cbd3fc43301f9b9987b5a8577d8fc0a14ba58 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Mon, 28 Apr 2025 20:10:57 +0000 Subject: [PATCH 01/24] preparing coco dataset with tags area and iscrowd --- supervision/dataset/core.py | 12 +++++++- supervision/dataset/formats/coco.py | 47 ++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index 8af54879..dc5de4d5 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -563,6 +563,8 @@ class DetectionDataset(BaseDataset): images_directory_path: str, annotations_path: str, force_masks: bool = False, + use_precomputed_area: bool = False, + use_iscrowd: bool = False, ) -> DetectionDataset: """ Creates a Dataset instance from COCO formatted data. @@ -574,7 +576,13 @@ class DetectionDataset(BaseDataset): force_masks (bool): If True, forces masks to be loaded for all annotations, regardless of whether they are present. - + use_precomputed_area (bool): If True, + uses precomputed area for all annotations, setting it to None if not + present. + use_iscrowd (bool): If True, + uses COCO's property `iscrowd` in all annotations, + regardless of whether they are present. If not presented, `iscrowd=0` + will be used. Returns: DetectionDataset: A DetectionDataset instance containing the loaded images and annotations. @@ -604,6 +612,8 @@ class DetectionDataset(BaseDataset): images_directory_path=images_directory_path, annotations_path=annotations_path, force_masks=force_masks, + use_precomputed_area=use_precomputed_area, + use_iscrowd=use_iscrowd, ) return DetectionDataset(classes=classes, images=images, annotations=annotations) diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index 9953d748..10f46d44 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -90,7 +90,11 @@ def coco_annotations_to_masks( def coco_annotations_to_detections( - image_annotations: List[dict], resolution_wh: Tuple[int, int], with_masks: bool + image_annotations: List[dict], + resolution_wh: Tuple[int, int], + with_masks: bool, + use_iscrowd: bool = False, + use_precomputed_area: bool = False, ) -> Detections: if not image_annotations: return Detections.empty() @@ -102,15 +106,32 @@ def coco_annotations_to_detections( xyxy = np.asarray(xyxy) xyxy[:, 2:4] += xyxy[:, 0:2] + if use_iscrowd: + iscrowd = [ + image_annotation["iscrowd"] for image_annotation in image_annotations + ] + else: + iscrowd = [0] * len(image_annotations) + + if use_precomputed_area: + area = [image_annotation["area"] for image_annotation in image_annotations] + else: + area = None + + data = dict( + iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float) + ) + if with_masks: mask = coco_annotations_to_masks( image_annotations=image_annotations, resolution_wh=resolution_wh ) - return Detections( - class_id=np.asarray(class_ids, dtype=int), xyxy=xyxy, mask=mask - ) + else: + mask = None - return Detections(xyxy=xyxy, class_id=np.asarray(class_ids, dtype=int)) + return Detections( + class_id=np.asarray(class_ids, dtype=int), xyxy=xyxy, mask=mask, data=data + ) def detections_to_coco_annotations( @@ -159,16 +180,29 @@ def detections_to_coco_annotations( return coco_annotations, annotation_id +def get_coco_class_index_mapping(annotations_path: str) -> Dict[int, int]: + coco_data = read_json_file(annotations_path) + classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) + class_mapping = build_coco_class_index_mapping( + coco_categories=coco_data["categories"], target_classes=classes + ) + return class_mapping + + def load_coco_annotations( images_directory_path: str, annotations_path: str, force_masks: bool = False, + use_iscrowd: bool = False, + use_precomputed_area: bool = False, ) -> Tuple[List[str], List[str], Dict[str, Detections]]: coco_data = read_json_file(file_path=annotations_path) classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) + class_index_mapping = build_coco_class_index_mapping( coco_categories=coco_data["categories"], target_classes=classes ) + coco_images = coco_data["images"] coco_annotations_groups = group_coco_annotations_by_image_id( coco_annotations=coco_data["annotations"] @@ -190,7 +224,10 @@ def load_coco_annotations( image_annotations=image_annotations, resolution_wh=(image_width, image_height), with_masks=force_masks, + use_iscrowd=use_iscrowd, + use_precomputed_area=use_precomputed_area, ) + annotation = map_detections_class_id( source_to_target_mapping=class_index_mapping, detections=annotation, From 232916a8ebf9afba2076d345aa85c7530ca48d98 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Mon, 28 Apr 2025 20:16:07 +0000 Subject: [PATCH 02/24] implementing mean average precision using COCO approach --- supervision/metrics/mean_average_precision.py | 1648 +++++++++++++---- 1 file changed, 1245 insertions(+), 403 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 9e7a30d0..17f4f394 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -1,422 +1,26 @@ from __future__ import annotations +import copy +import datetime +import itertools +from collections import defaultdict from copy import deepcopy from dataclasses import dataclass -from typing import TYPE_CHECKING, List, Optional, Tuple, Union +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np from matplotlib import pyplot as plt -from supervision.config import ORIENTED_BOX_COORDINATES from supervision.detection.core import Detections -from supervision.detection.utils import ( - box_iou_batch, - mask_iou_batch, - oriented_box_iou_batch, -) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget -from supervision.metrics.utils.object_size import ( - ObjectSizeCategory, - get_detection_size_category, -) from supervision.metrics.utils.utils import ensure_pandas_installed if TYPE_CHECKING: import pandas as pd -class MeanAveragePrecision(Metric): - """ - Mean Average Precision (mAP) is a metric used to evaluate object detection models. - It is the average of the precision-recall curves at different IoU thresholds. - - Example: - ```python - import supervision as sv - from supervision.metrics import MeanAveragePrecision - - predictions = sv.Detections(...) - targets = sv.Detections(...) - - map_metric = MeanAveragePrecision() - map_result = map_metric.update(predictions, targets).compute() - - print(map_result.map50_95) - # 0.4674 - - print(map_result) - # MeanAveragePrecisionResult: - # Metric target: MetricTarget.BOXES - # Class agnostic: False - # mAP @ 50:95: 0.4674 - # mAP @ 50: 0.5048 - # mAP @ 75: 0.4796 - # mAP scores: [0.50485 0.50377 0.50377 ...] - # IoU thresh: [0.5 0.55 0.6 ...] - # AP per class: - # 0: [0.67699 0.67699 0.67699 ...] - # ... - # Small objects: ... - # Medium objects: ... - # Large objects: ... - - map_result.plot() - ``` - - ![example_plot](\ - https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\ - ){ align=center width="800" } - """ - - def __init__( - self, - metric_target: MetricTarget = MetricTarget.BOXES, - class_agnostic: bool = False, - ): - """ - Initialize the Mean Average Precision metric. - - Args: - metric_target (MetricTarget): The type of detection data to use. - class_agnostic (bool): Whether to treat all data as a single class. - """ - self._metric_target = metric_target - self._class_agnostic = class_agnostic - - self._predictions_list: List[Detections] = [] - self._targets_list: List[Detections] = [] - - def reset(self) -> None: - """ - Reset the metric to its initial state, clearing all stored data. - """ - self._predictions_list = [] - self._targets_list = [] - - def update( - self, - predictions: Union[Detections, List[Detections]], - targets: Union[Detections, List[Detections]], - ) -> MeanAveragePrecision: - """ - Add new predictions and targets to the metric, but do not compute the result. - - Args: - predictions (Union[Detections, List[Detections]]): The predicted detections. - targets (Union[Detections, List[Detections]]): The ground-truth detections. - - Returns: - (MeanAveragePrecision): The updated metric instance. - """ - if not isinstance(predictions, list): - predictions = [predictions] - if not isinstance(targets, list): - targets = [targets] - - if len(predictions) != len(targets): - raise ValueError( - f"The number of predictions ({len(predictions)}) and" - f" targets ({len(targets)}) during the update must be the same." - ) - - if self._class_agnostic: - predictions = deepcopy(predictions) - targets = deepcopy(targets) - - for prediction in predictions: - prediction.class_id[:] = -1 - for target in targets: - target.class_id[:] = -1 - - self._predictions_list.extend(predictions) - self._targets_list.extend(targets) - - return self - - def compute( - self, - ) -> MeanAveragePrecisionResult: - """ - Calculate Mean Average Precision based on predicted and ground-truth - detections at different thresholds. - - Returns: - (MeanAveragePrecisionResult): The Mean Average Precision result. - """ - result = self._compute(self._predictions_list, self._targets_list) - - small_predictions = [] - small_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - small_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL) - ) - small_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL) - ) - result.small_objects = self._compute(small_predictions, small_targets) - - medium_predictions = [] - medium_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - medium_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM) - ) - medium_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM) - ) - result.medium_objects = self._compute(medium_predictions, medium_targets) - - large_predictions = [] - large_targets = [] - for predictions, targets in zip(self._predictions_list, self._targets_list): - large_predictions.append( - self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE) - ) - large_targets.append( - self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE) - ) - result.large_objects = self._compute(large_predictions, large_targets) - - return result - - def _compute( - self, - predictions_list: List[Detections], - targets_list: List[Detections], - ) -> MeanAveragePrecisionResult: - iou_thresholds = np.linspace(0.5, 0.95, 10) - stats = [] - - for predictions, targets in zip(predictions_list, targets_list): - prediction_contents = self._detections_content(predictions) - target_contents = self._detections_content(targets) - - if len(targets) > 0: - if len(predictions) == 0: - stats.append( - ( - np.zeros((0, iou_thresholds.size), dtype=bool), - np.zeros((0,), dtype=np.float32), - np.zeros((0,), dtype=int), - targets.class_id, - ) - ) - - else: - if self._metric_target == MetricTarget.BOXES: - iou = box_iou_batch(target_contents, prediction_contents) - elif self._metric_target == MetricTarget.MASKS: - iou = mask_iou_batch(target_contents, prediction_contents) - elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - iou = oriented_box_iou_batch( - target_contents, prediction_contents - ) - else: - raise ValueError( - "Unsupported metric target for IoU calculation" - ) - - matches = self._match_detection_batch( - predictions.class_id, targets.class_id, iou, iou_thresholds - ) - - stats.append( - ( - matches, - predictions.confidence, - predictions.class_id, - targets.class_id, - ) - ) - - # Compute average precisions if any matches exist - if stats: - concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)] - average_precisions, unique_classes = self._average_precisions_per_class( - *concatenated_stats - ) - mAP_scores = np.mean(average_precisions, axis=0) - else: - mAP_scores = np.zeros((10,), dtype=np.float32) - unique_classes = np.empty((0,), dtype=int) - average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32) - - return MeanAveragePrecisionResult( - metric_target=self._metric_target, - is_class_agnostic=self._class_agnostic, - mAP_scores=mAP_scores, - iou_thresholds=iou_thresholds, - matched_classes=unique_classes, - ap_per_class=average_precisions, - ) - - @staticmethod - def _compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float: - """ - Compute the average precision using 101-point interpolation (COCO), given - the recall and precision curves. - - Args: - recall (np.ndarray): The recall curve. - precision (np.ndarray): The precision curve. - - Returns: - (float): Average precision. - """ - if len(recall) == 0 and len(precision) == 0: - return 0.0 - - recall_levels = np.linspace(0, 1, 101) - precision_levels = np.zeros_like(recall_levels) - for r, p in zip(recall[::-1], precision[::-1]): - precision_levels[recall_levels <= r] = p - - average_precision = (1 / 101 * precision_levels).sum() - return average_precision - - @staticmethod - def _match_detection_batch( - predictions_classes: np.ndarray, - target_classes: np.ndarray, - iou: np.ndarray, - iou_thresholds: np.ndarray, - ) -> np.ndarray: - num_predictions, num_iou_levels = ( - predictions_classes.shape[0], - iou_thresholds.shape[0], - ) - correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) - correct_class = target_classes[:, None] == predictions_classes - - for i, iou_level in enumerate(iou_thresholds): - matched_indices = np.where((iou >= iou_level) & correct_class) - - if matched_indices[0].shape[0]: - combined_indices = np.stack(matched_indices, axis=1) - iou_values = iou[matched_indices][:, None] - matches = np.hstack([combined_indices, iou_values]) - - if matched_indices[0].shape[0] > 1: - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] - - correct[matches[:, 1].astype(int), i] = True - - return correct - - @staticmethod - def _average_precisions_per_class( - matches: np.ndarray, - prediction_confidence: np.ndarray, - prediction_class_ids: np.ndarray, - true_class_ids: np.ndarray, - ) -> Tuple[np.ndarray, np.ndarray]: - """ - Compute the average precision, given the recall and precision curves. - Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. - - Args: - matches (np.ndarray): True positives. - prediction_confidence (np.ndarray): Objectness value from 0-1. - prediction_class_ids (np.ndarray): Predicted object classes. - true_class_ids (np.ndarray): True object classes. - eps (float, optional): Small value to prevent division by zero. - - Returns: - (Tuple[np.ndarray, np.ndarray]): Average precision for different - IoU levels, and an array of class IDs that were matched. - """ - eps = 1e-16 - - sorted_indices = np.argsort(-prediction_confidence) - matches = matches[sorted_indices] - prediction_class_ids = prediction_class_ids[sorted_indices] - - unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) - num_classes = unique_classes.shape[0] - - average_precisions = np.zeros((num_classes, matches.shape[1])) - - for class_idx, class_id in enumerate(unique_classes): - is_class = prediction_class_ids == class_id - total_true = class_counts[class_idx] - total_prediction = is_class.sum() - - if total_prediction == 0 or total_true == 0: - continue - - false_positives = (1 - matches[is_class]).cumsum(0) - true_positives = matches[is_class].cumsum(0) - false_negatives = total_true - true_positives - - recall = true_positives / (true_positives + false_negatives + eps) - precision = true_positives / (true_positives + false_positives) - - for iou_level_idx in range(matches.shape[1]): - average_precisions[class_idx, iou_level_idx] = ( - MeanAveragePrecision._compute_average_precision( - recall[:, iou_level_idx], precision[:, iou_level_idx] - ) - ) - - return average_precisions, unique_classes - - def _detections_content(self, detections: Detections) -> np.ndarray: - """Return boxes, masks or oriented bounding boxes from detections.""" - if self._metric_target == MetricTarget.BOXES: - return detections.xyxy - if self._metric_target == MetricTarget.MASKS: - return ( - detections.mask - if detections.mask is not None - else self._make_empty_content() - ) - if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - obb = detections.data.get(ORIENTED_BOX_COORDINATES) - if obb is not None and len(obb) > 0: - return np.array(obb, dtype=np.float32) - return self._make_empty_content() - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _make_empty_content(self) -> np.ndarray: - if self._metric_target == MetricTarget.BOXES: - return np.empty((0, 4), dtype=np.float32) - if self._metric_target == MetricTarget.MASKS: - return np.empty((0, 0, 0), dtype=bool) - if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - return np.empty((0, 4, 2), dtype=np.float32) - raise ValueError(f"Invalid metric target: {self._metric_target}") - - def _filter_detections_by_size( - self, detections: Detections, size_category: ObjectSizeCategory - ) -> Detections: - """Return a copy of detections with contents filtered by object size.""" - new_detections = deepcopy(detections) - if detections.is_empty() or size_category == ObjectSizeCategory.ANY: - return new_detections - - sizes = get_detection_size_category(new_detections, self._metric_target) - size_mask = sizes == size_category.value - - new_detections.xyxy = new_detections.xyxy[size_mask] - if new_detections.mask is not None: - new_detections.mask = new_detections.mask[size_mask] - if new_detections.class_id is not None: - new_detections.class_id = new_detections.class_id[size_mask] - if new_detections.confidence is not None: - new_detections.confidence = new_detections.confidence[size_mask] - if new_detections.tracker_id is not None: - new_detections.tracker_id = new_detections.tracker_id[size_mask] - if new_detections.data is not None: - for key, value in new_detections.data.items(): - new_detections.data[key] = np.array(value)[size_mask] - - return new_detections - - @dataclass class MeanAveragePrecisionResult: """ @@ -553,7 +157,6 @@ class MeanAveragePrecisionResult: pandas_data[f"large_objects_{key}"] = value # Average precisions are currently not included in the DataFrame. - return pd.DataFrame( pandas_data, index=[0], @@ -626,3 +229,1242 @@ class MeanAveragePrecisionResult: plt.tight_layout() plt.show() + + def to_pycocotools_output(self) -> str: + """ + Convert the result to the same output format as pycocotools. + """ + return ( + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " + f"maxDets=100 ] = {self.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50 | area= all | " + f"maxDets=100 ] = {self.map50:.3f}\n" + f" Average Precision (AP) @[ IoU=0.75 | area= all | " + f"maxDets=100 ] = {self.map75:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = {self.small_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = {self.medium_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = {self.large_objects.map50_95:.3f}" + ) + + +class EvaluationDataset: + """ + Class used to evaluate models with Mean Average Precision. + """ + + def __init__(self, targets: Optional[Dict[str, Any]] = None): + """ + Constructor of EvaluationDataset object used to evaluate models with + Mean Average Precision. + Args: + targets (dict): The targets (ground truth) of the dataset in a the + COCO format. + """ + # Initialize members + self.dataset, self.anns, self.cats, self.imgs = dict(), dict(), dict(), dict() + self.img_to_anns, self.cat_to_imgs = defaultdict(list), defaultdict(list) + + if targets is None: + return + + # Load dataset + self.dataset = targets + self.create_class_members() + + @classmethod + def empty(cls): + return cls(targets=None) + + def create_class_members(self): + """ + Create index elements for the dataset. + """ + anns, cats, imgs = {}, {}, {} + img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list) + if "annotations" in self.dataset: + for ann in self.dataset["annotations"]: + img_to_anns[ann["image_id"]].append(ann) + anns[ann["id"]] = ann + + if "images" in self.dataset: + for img in self.dataset["images"]: + imgs[img["id"]] = img + + if "categories" in self.dataset: + for cat in self.dataset["categories"]: + cats[cat["id"]] = cat + + if "annotations" in self.dataset and "categories" in self.dataset: + for ann in self.dataset["annotations"]: + cat_to_imgs[ann["category_id"]].append(ann["image_id"]) + + # Populate class members + self.anns = anns + self.img_to_anns = img_to_anns + self.cat_to_imgs = cat_to_imgs + self.imgs = imgs + self.cats = cats + + def get_annotation_ids( + self, + img_ids: List[int] = [], + cat_ids: List[int] = [], + area_range: Tuple[float, float] = [], + iscrowd: bool = False, + ): + """ + Get annotation ids that satisfy given filter conditions. + Args: + img_ids (list): ids of the images that we want to retrieve. + cat_ids (list): ids of the categories that we want to retrieve. + area_range (tuple): area range of the annotations that we want to retrieve in + the format [min_area, max_area]. + iscrowd (bool): if annotations to retrieve are `iscrowded=1`. + """ + # If there are no filters, we use all annotations + if len(img_ids) == len(cat_ids) == len(area_range) == 0: + anns = self.dataset["annotations"] + else: + if len(img_ids) != 0: + lists = [ + self.img_to_anns[img_id] + for img_id in img_ids + if img_id in self.img_to_anns + ] + anns = list(itertools.chain.from_iterable(lists)) + else: + anns = self.dataset["annotations"] + + # Filter by category + anns = ( + anns + if len(cat_ids) == 0 + else [ann for ann in anns if ann["category_id"] in cat_ids] + ) + + # Filter by area + anns = ( + anns + if len(area_range) == 0 + else [ + ann + for ann in anns + if ann["area"] > area_range[0] and ann["area"] < area_range[1] + ] + ) + + # Filter by iscrowd + if iscrowd is True: + ids = [ann["id"] for ann in anns if ann["iscrowd"] == 1] + else: + ids = [ann["id"] for ann in anns] + return ids + + def get_category_ids( + self, + cat_names: List[str] = [], + supercategory_names: List[str] = [], + cat_ids: List[int] = [], + ) -> List[int]: + """ + Get category ids that satisfy given filter conditions. + Args: + cat_names (list): names of the categories to retrieve. + supercategory_names (list): names of the supercategories to retrieve. + cat_ids (list): ids of the categories to retrieve. + Returns: + ids (list): integer array of category ids. + """ + # If there are no filters, we use all categories + if len(cat_names) == len(supercategory_names) == len(cat_ids) == 0: + cats = self.dataset["categories"] + else: + cats = self.dataset["categories"] + + # Filter by name + cats = ( + cats + if len(cat_names) == 0 + else [cat for cat in cats if cat["name"] in cat_names] + ) + + # Filter by supercategory + cats = ( + cats + if len(supercategory_names) == 0 + else [ + cat for cat in cats if cat["supercategory"] in supercategory_names + ] + ) + + # Filter by id + cats = ( + cats + if len(cat_ids) == 0 + else [cat for cat in cats if cat["id"] in cat_ids] + ) + ids = [cat["id"] for cat in cats] + return ids + + def get_image_ids( + self, + img_ids: List[int] = [], + cat_ids: List[int] = [], + ) -> List[int]: + """ + Get image ids that satisfy given filter conditions. + Args: + img_ids (list): ids of the images to retrieve. + cat_ids (list): ids of the categories to retrieve. + Returns: + ids (list): integer array of image ids. + """ + # If there are no filters, we use all images + if len(img_ids) == len(cat_ids) == 0: + ids = self.imgs.keys() + return list(ids) + + ids = set(img_ids) + for i, cat_id in enumerate(cat_ids): + if i == 0 and len(ids) == 0: + ids = set(self.cat_to_imgs[cat_id]) + else: + ids &= set(self.cat_to_imgs[cat_id]) + return list(ids) + + def get_annotations(self, ids: List[int] = []) -> List[dict]: + """ + Get annotations with the specified ids. + Args: + ids (list): integer ids specifying annotations. + Returns: + anns (list): loaded annotations. + """ + return [self.anns[idx] for idx in ids] + + def load_predictions(self, predictions: List[Dict]) -> "EvaluationDataset": + """ + Load prediction result into an EvaluationDataset object. + Args: + predictions (list): prediction result. + Returns: + EvaluationDataset object representing the predictions. + """ + # Create an empty EvaluationDataset object for the predictions + predictions_dataset = EvaluationDataset.empty() + predictions_dataset.dataset["images"] = [img for img in self.dataset["images"]] + + if not isinstance(predictions, list): + raise ValueError("results must be a list") + + ids = [pred["image_id"] for pred in predictions] + + # Make sure the image ids from predictions exist in the current dataset + assert set(ids) == (set(ids) & set(self.get_image_ids())), ( + "Results do not correspond to current coco set" + ) + + # Check if the predictions contain any unsupported keys + if "caption" in predictions[0]: + raise NotImplementedError( + "Evaluating predictions with caption is not supported." + ) + elif "segmentation" in predictions[0]: + raise NotImplementedError( + "Evaluating predictions with segmentation is not supported." + ) + elif "keypoints" in predictions[0]: + raise NotImplementedError( + "Evaluating predictions with keypoints is not supported." + ) + + elif "bbox" in predictions[0] and not predictions[0]["bbox"] == []: + predictions_dataset.dataset["categories"] = copy.deepcopy( + self.dataset["categories"] + ) + + # Prepare fields for every prediction of the given image + for idx, pred in enumerate(predictions): + x, y, w, h = pred["bbox"] + x1, x2, y1, y2 = [x, x + w, y, y + h] + + # Make segmentation from bounding box coordinates + if "segmentation" not in pred: + pred["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]] + pred["area"] = w * h + pred["id"] = idx + 1 + # For predictions we set iscrowd to 0 + pred["iscrowd"] = 0 + predictions_dataset.dataset["annotations"] = predictions + predictions_dataset.create_class_members() + return predictions_dataset + + +# Area ranges for object size in pixels +SMALL_OBJECT_AREA = 32**2 +MEDIUM_OBJECT_AREA = 96**2 +MAX_ALL_OBJECT_AREA = 1e5**2 + +# Smallest number to avoid division by zero +EPS = np.spacing(1) + + +class ObjectSize(Enum): + """ + Enum for object size. + """ + + ALL = "all" + SMALL = "small" + MEDIUM = "medium" + LARGE = "large" + + +def _iou_with_jaccard( + dt: List[List[float]], gt: List[List[float]], is_crowd: List[bool] +) -> np.ndarray: + """ + Calculate the intersection over union (IoU) between detection bounding boxes (dt) + and ground-truth bounding boxes (gt). + Reference: https://github.com/rafaelpadilla/review_object_detection_metrics + + Args: + dt (List[List[float]]): List of detection bounding boxes in the \ + format [x, y, width, height]. + gt (List[List[float]]): List of ground-truth bounding boxes in the \ + format [x, y, width, height]. + is_crowd (List[bool]): List indicating if each ground-truth bounding box \ + is a crowd region or not. + + Returns: + np.ndarray: Array of IoU values of shape (len(dt), len(gt)). + """ + assert len(is_crowd) == len(gt), "iou(iscrowd=) must have the same length as gt" + if len(dt) == 0 or len(gt) == 0: + return np.array([]) + ious = np.zeros((len(dt), len(gt)), dtype=np.float64) + for g_idx, g in enumerate(gt): + for d_idx, d in enumerate(dt): + ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) + return ious + + +def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: + """ + Calculate the Jaccard index (intersection over union) between two bounding boxes. + If a gt object is marked as "iscrowd", a dt is allowed to match any subregion + of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using + gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing + iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) + + Args: + box_a (List[float]): Box coordinates in the format [x, y, width, height]. + box_b (List[float]): Box coordinates in the format [x, y, width, height]. + iscrowd (bool): Flag indicating if the second box is a crowd region or not. + + Returns: + float: Jaccard index between the two bounding boxes. + """ + xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] + xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] + + # Innermost left x + xi = max(xa, xb) + # Innermost right x + x2i = min(x2a, x2b) + # Same for y + yi = max(ya, yb) + y2i = min(y2a, y2b) + + # Calculate areas + Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) + Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) + Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) + + if is_crowd: + return Ai / (Aa + EPS) + + return Ai / (Aa + Ab - Ai + EPS) + + +class COCOEvaluatorParameters: + """ + Parameters for COCOEvaluator + """ + + def __init__(self): + """Initialize all parameters for evaluation""" + + self.img_ids, self.cat_ids = [], [] + # IoU thresholds [0.5, 0.55, 0.6, 0.65, ..., 0.95] + self.iou_thrs = np.linspace( + 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True + ) + # 101 recall thresholds [0.0, 0.01, 0.02, ..., 1.00] + self.rec_thrs = np.linspace( + 0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True + ) + # 3 maximum detection thresholds [1, 10, 100] + self.max_dets = [1, 10, 100] + # Area ranges [0, 1e5], [0, 32], [32, 96], [96, 1e5] + self.area_range = [ + [0, MAX_ALL_OBJECT_AREA], + [0, SMALL_OBJECT_AREA], + [SMALL_OBJECT_AREA, MEDIUM_OBJECT_AREA], + [MEDIUM_OBJECT_AREA, MAX_ALL_OBJECT_AREA], + ] + + +class COCOEvaluator: + def __init__( + self, coco_targets: EvaluationDataset, coco_predictions: EvaluationDataset + ): + """ + Constructor of COCOEvaluator object. + Args: + coco_targets (EvaluationDataset): The dataset with the ground truths. + coco_predictions (EvaluationDataset): The dataset with the predictions. + """ + if coco_targets is None: + raise ValueError("coco_targets must be provided") + if coco_predictions is None: + raise ValueError("coco_predictions must be provided") + + self.coco_targets = coco_targets + self.coco_predictions = coco_predictions + # List of dictionaries containing the evaluation results + # len(eval_imgs) = (categories) * (area_ranges) * (images) + # For COCO 2017: len(eval_images) = 80 * 4 * 5000 = 1600000 + self.eval_imgs = defaultdict(list) + # Dictionary of accumulated results + self.results = {} + # Dictionary of targets for evaluation + self._targets = defaultdict(list) + self._predictions = defaultdict(list) + # Parameters for evaluation + self.params = COCOEvaluatorParameters() + # List of results summarization + self.stats = [] + # Dictionary of IOUs between all targets and predictions + self.ious = {} + # Set image and category ids + self.params.img_ids = sorted(self.coco_targets.get_image_ids()) + self.params.cat_ids = sorted(self.coco_targets.get_category_ids()) + + def _prepare_targets_and_predictions(self): + """ + Prepare targets and predictions for evaluation. + """ + # Get the target samples for the evaluation + annotation_ids = self.coco_targets.get_annotation_ids( + img_ids=self.params.img_ids, cat_ids=self.params.cat_ids + ) + targets = self.coco_targets.get_annotations(annotation_ids) + # Get the prediction samples for the evaluation + prediction_ids = self.coco_predictions.get_annotation_ids( + img_ids=self.params.img_ids, cat_ids=self.params.cat_ids + ) + predictions = self.coco_predictions.get_annotations(prediction_ids) + + # Set ignore flag + for gt in targets: + gt["ignore"] = gt["ignore"] if "ignore" in gt else 0 + gt["ignore"] = "iscrowd" in gt and gt["iscrowd"] + + # Select targets + self._targets = defaultdict(list) + for gt in targets: + self._targets[gt["image_id"], gt["category_id"]].append(gt) + + # Select predictions + self._predictions = defaultdict(list) + for dt in predictions: + self._predictions[dt["image_id"], dt["category_id"]].append(dt) + + # Initialize evaluation results + self.eval_imgs = defaultdict(list) + self.results = {} + + def _compute_iou(self, img_id: int, cat_id: int) -> np.ndarray: + """ + Compute the IoU between the targets and predictions for a given image and + category. + + Args: + img_id (int): The image id. + cat_id (int): The category id. + + Returns: + np.ndarray: The IoU between the targets and predictions. + """ + gt = self._targets[img_id, cat_id] + dt = self._predictions[img_id, cat_id] + + # If there is nothing to evaluate + if len(gt) == 0 and len(dt) == 0: + return np.array([]) + + # Sort predictions by highest score first + inds = np.argsort([-d["score"] for d in dt], kind="stable") + dt = [dt[i] for i in inds] + + # Truncate the predictions if there are more predictions than the max detections + # to evaluate + if len(dt) > self.params.max_dets[-1]: + dt = dt[0 : self.params.max_dets[-1]] + + gt_boxes = [g["bbox"] for g in gt] + dt_boxes = [d["bbox"] for d in dt] + + # Get the iscrowd flag for each gt + is_crowd = [int(o["iscrowd"]) for o in gt] + # Compute iou between each prediction a and gt region + iou = _iou_with_jaccard(dt_boxes, gt_boxes, is_crowd) + return iou + + def _evaluate_image( + self, img_id: int, cat_id: int, area_range: Tuple[int, int], max_det: int + ) -> Union[Dict[str, Any], None]: + """ + Perform evaluation for single category and image. + Args: + img_id (int): The image id. + cat_id (int): The category id. + area_range (Tuple[int, int]): The area range. + max_det (int): The maximum number of detections. + + Returns: + Dict[str, Any]: The evaluation results. + """ + # Get targets (gt) and predictions (dt) for the given image and category + gt = self._targets[img_id, cat_id] + dt = self._predictions[img_id, cat_id] + + # If there is nothing to evaluate + if len(gt) == 0 and len(dt) == 0: + return None + + min_area, max_area = area_range + + # Create an `_ignore` flag for targets if they are set as ignore or their area + # is not in the range [min_area, max_area] + for g in gt: + if g["ignore"] or not (min_area <= g["area"] <= max_area): + g["_ignore"] = 1 + else: + g["_ignore"] = 0 + + # Sort ground-truths by ignore flag (0: non ignored, 1: ignored) + gt_sorted = np.argsort([g["_ignore"] for g in gt], kind="stable") + gt = [gt[i] for i in gt_sorted] + + # Sort predictions by scores in descending order + dt_sorted = np.argsort([-d["score"] for d in dt], kind="stable") + dt = [dt[i] for i in dt_sorted[0:max_det]] + + # Get the iscrowd flag for each gt + # iscrowd = [int(o["iscrowd"]) for o in gt] + + # Load computed ious for the given image and category + ious = ( + self.ious[img_id, cat_id][:, gt_sorted] + if len(self.ious[img_id, cat_id]) > 0 + else self.ious[img_id, cat_id] + ) + + # Get the number of thresholds, ground truths and detections + num_thresholds = len(self.params.iou_thrs) + num_ground_truths = len(gt) + num_detections = len(dt) + + # Initialize matches: 0 means no match + gt_matches = np.zeros((num_thresholds, num_ground_truths)) + dt_matches = np.zeros((num_thresholds, num_detections)) + # Initialize ignore flags: 0 means no ignore + gt_ignore = np.array([g["_ignore"] for g in gt]) + dt_ignore = np.zeros((num_thresholds, num_detections)) + if len(ious) != 0: + # Go through the iou thresholds + for tresh_idx, thresh in enumerate(self.params.iou_thrs): + # Go through the detections + for det_idx, det in enumerate(dt): + # Start the iou of the best match + iou_best_match = min([thresh, 1 - 1e-10]) + # Set the best match index to -1 (unmatched) + best_match_idx = -1 + # Go through the ground truths + for g_idx, g in enumerate(gt): + # If current gt is already matched, and not a crowd, continue + # if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd[g_idx]: + iscrowd = int(g.get("iscrowd")) + if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd: + continue + # Stop searching the ground truths + if ( + best_match_idx > -1 # detection is matched to a gt + and gt_ignore[best_match_idx] + == 0 # matched gt is not ignored + and gt_ignore[g_idx] == 1 # current gt is ignored + ): + break + + # A new best match was found + if ious[det_idx, g_idx] >= iou_best_match: + iou_best_match = ious[det_idx, g_idx] + best_match_idx = g_idx + + # A best match was found + if best_match_idx != -1: + dt_ignore[tresh_idx, det_idx] = gt_ignore[best_match_idx] + dt_matches[tresh_idx, det_idx] = gt[best_match_idx]["id"] + gt_matches[tresh_idx, best_match_idx] = det["id"] + + # Set unmatched detections outside of area range to ignore + area_range_mask = np.array( + [d["area"] < min_area or d["area"] > max_area for d in dt] + ).reshape((1, len(dt))) + + # Update the ignore flags for detections + dt_ignore = np.logical_or( + dt_ignore, + np.logical_and( + dt_matches == 0, np.repeat(area_range_mask, num_thresholds, 0) + ), + ) + + return { + "image_id": img_id, + "category_id": cat_id, + "area_range": area_range, + "max_det": max_det, + "dt_ids": [d["id"] for d in dt], + "gt_ids": [g["id"] for g in gt], + "dtMatches": dt_matches, + "gtMatches": gt_matches, + "dtScores": [d["score"] for d in dt], + "gtIgnore": gt_ignore, + "dtIgnore": dt_ignore, + } + + def __str__(self): + self.summarize() + + def _accumulate(self): + """ + Accumulate per image evaluation results and store the result in self.results + """ + # Get the number of thresholds, categories, area ranges, and max detections + num_iou_thresholds = len(self.params.iou_thrs) + num_recall_thresholds = len(self.params.rec_thrs) + num_categories = len(self.params.cat_ids) + num_area_ranges = len(self.params.area_range) + num_max_detections = len(self.params.max_dets) + num_imgs = len(self.params.img_ids) + + # Initialize precision, recall, and scores arrays + # -1 means absent categories + precision = -np.ones( + ( + num_iou_thresholds, + num_recall_thresholds, + num_categories, + num_area_ranges, + num_max_detections, + ) + ) + recall = -np.ones( + (num_iou_thresholds, num_categories, num_area_ranges, num_max_detections) + ) + scores = -np.ones( + ( + num_iou_thresholds, + num_recall_thresholds, + num_categories, + num_area_ranges, + num_max_detections, + ) + ) + + # Create sets for indexing + set_categories = set(self.params.cat_ids) + set_area_ranges = set(map(tuple, self.params.area_range)) + set_max_detections = set(self.params.max_dets) + set_image_ids = set(self.params.img_ids) + + # Select category indexes to evaluate + selected_category_ids = [ + n for n, k in enumerate(self.params.cat_ids) if k in set_categories + ] + # Select max detections to evaluate + selected_max_detections = [ + m for m in self.params.max_dets if m in set_max_detections + ] + # Select area ranges to evaluate + selected_area_ranges_ids = [ + idx + for idx, area in enumerate(self.params.area_range) + if tuple(area) in set_area_ranges + ] + # Select image indexes to evaluate + image_inds = [ + n for n, i in enumerate(self.params.img_ids) if i in set_image_ids + ] + + # Evaluting at all categories, area ranges, max number of detections, and + # IoU thresholds + + # Loop through categories + for cat_idx, cat_eval_idx in enumerate(selected_category_ids): + cat_offset = cat_eval_idx * num_area_ranges * num_imgs + + # Loop through area ranges + for area_idx, area_eval_idx in enumerate(selected_area_ranges_ids): + area_offset = area_eval_idx * num_imgs + + # Loop through max detections + for max_det_idx, max_det in enumerate(selected_max_detections): + eval_img_data = [ + self.eval_imgs[cat_offset + area_offset + i] for i in image_inds + ] + eval_img_data = [e for e in eval_img_data if e is not None] + + # No image to evaluate + if len(eval_img_data) == 0: + continue + + # Sort detected scores in descending order + dt_scores = np.concatenate( + [e["dtScores"][0:max_det] for e in eval_img_data] + ) + inds = np.argsort(-dt_scores, kind="stable") + dt_scores_sorted = dt_scores[inds] + + # Get matches and ignored matches + dt_matches = np.concatenate( + [e["dtMatches"][:, 0:max_det] for e in eval_img_data], axis=1 + )[:, inds] + dt_ignored = np.concatenate( + [e["dtIgnore"][:, 0:max_det] for e in eval_img_data], axis=1 + )[:, inds] + + # Get ignored ground truth objects + gt_ignored = np.concatenate([e["gtIgnore"] for e in eval_img_data]) + num_non_ignored_gt = np.count_nonzero(gt_ignored == 0) + + # No ground truth objects to evaluate + if num_non_ignored_gt == 0: + continue + + # Compute true positives and false positives + true_positives = np.logical_and( + dt_matches, np.logical_not(dt_ignored) + ) + false_positives = np.logical_and( + np.logical_not(dt_matches), np.logical_not(dt_ignored) + ) + + tp_sum = np.cumsum(true_positives, axis=1).astype(dtype=np.float64) + fp_sum = np.cumsum(false_positives, axis=1).astype(dtype=np.float64) + + # Loop through thresholds + for iou_thresh_idx, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + num_tps = len(tp) + # Recall: TP / Total number of ground truth objects + rc = tp / num_non_ignored_gt + # Precision: TP / (FP + TP) + pr = (tp / (fp + tp + EPS)).tolist() + # List to compute the precision at each recall threshold + precision_at_recall = [0] * num_recall_thresholds + # List to compute the score at each recall threshold + score_at_recall = [0] * num_recall_thresholds + + # Set recall to either the final recall value or 0 (when there + # is no TP) + recall[iou_thresh_idx, cat_idx, area_idx, max_det_idx] = ( + rc[-1] if num_tps else 0 + ) + + # Loop through precision values + for i in range(num_tps - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + inds = np.searchsorted(rc, self.params.rec_thrs, side="left") + for ri, pos_idx in enumerate(inds): + # Ensure pi is within the range of both arrays before using it + if 0 <= pos_idx < len(pr) and 0 <= pos_idx < len( + dt_scores_sorted + ): + precision_at_recall[ri] = pr[pos_idx] + score_at_recall[ri] = dt_scores_sorted[pos_idx] + + # Convert precision to numpy array + precision[iou_thresh_idx, :, cat_idx, area_idx, max_det_idx] = ( + np.array(precision_at_recall) + ) + # Convert scores to numpy array + scores[iou_thresh_idx, :, cat_idx, area_idx, max_det_idx] = ( + np.array(score_at_recall) + ) + + # Average precision over all sizes, 100 max detections + area_range_idx = list(ObjectSize).index(ObjectSize.ALL) + max_100_dets_idx = self.params.max_dets.index(100) + # Average precision [threshold, recall, classes] + average_precision_all_sizes = precision[ + :, :, :, area_range_idx, max_100_dets_idx + ] + # mAP over thresholds (dimension=num_thresholds) + mAP_scores_all_sizes = average_precision_all_sizes.mean(axis=(1, 2)) + # AP per class + ap_per_class_all_sizes = average_precision_all_sizes.mean(axis=1).transpose( + 1, 0 + ) + + # Average precision for SMALL objects and 100 max detections + small_area_range_idx = list(ObjectSize).index(ObjectSize.SMALL) + average_precision_small = precision[ + :, :, :, small_area_range_idx, max_100_dets_idx + ] + mAP_scores_small = average_precision_small.mean(axis=(1, 2)) + ap_per_class_small = average_precision_small.mean(axis=1).transpose(1, 0) + + # Average precision for MEDIUM objects and 100 max detections + medium_area_range_idx = list(ObjectSize).index(ObjectSize.MEDIUM) + average_precision_medium = precision[ + :, :, :, medium_area_range_idx, max_100_dets_idx + ] + mAP_scores_medium = average_precision_medium.mean(axis=(1, 2)) + ap_per_class_medium = average_precision_medium.mean(axis=1).transpose(1, 0) + + # Average precision for LARGE objects and 100 max detections + large_area_range_idx = list(ObjectSize).index(ObjectSize.LARGE) + average_precision_large = precision[ + :, :, :, large_area_range_idx, max_100_dets_idx + ] + mAP_scores_large = average_precision_large.mean(axis=(1, 2)) + ap_per_class_large = average_precision_large.mean(axis=1).transpose(1, 0) + + self.results = { + "params": self.params, + "counts": [ + num_iou_thresholds, + num_recall_thresholds, + num_categories, + num_area_ranges, + num_max_detections, + ], + "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "precision": precision, + "recall": recall, + "scores": scores, + "mAP_scores_all_sizes": mAP_scores_all_sizes, + "ap_per_class_all_sizes": ap_per_class_all_sizes, + "mAP_scores_small": mAP_scores_small, + "ap_per_class_small": ap_per_class_small, + "mAP_scores_medium": mAP_scores_medium, + "ap_per_class_medium": ap_per_class_medium, + "mAP_scores_large": mAP_scores_large, + "ap_per_class_large": ap_per_class_large, + } + + def _pycocotools_summarize(self): + """ + Compute and display summary metrics for evaluation results. + """ + + def _summarize( + use_ap: bool = True, iou_thr=None, area_range=ObjectSize.ALL, max_dets=100 + ): + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.10f}" + titleStr = "Average Precision" if use_ap else "Average Recall" + typeStr = "(AP)" if use_ap else "(AR)" + iou_str = ( + "{:0.2f}:{:0.2f}".format( + self.params.iou_thrs[0], self.params.iou_thrs[-1] + ) + if iou_thr is None + else "{:0.2f}".format(iou_thr) + ) + all_object_sizes = list(ObjectSize) + area_range_idx = all_object_sizes.index(area_range) + max_detections_idx = self.params.max_dets.index(max_dets) + if use_ap: + # Dimension of precision: + # threshold x recall x classes x areas x max detections + s = self.results["precision"] + # IOU + if iou_thr is not None: + t = np.where(iou_thr == self.params.iou_thrs)[0] + s = s[t] + s = s[:, :, :, area_range_idx, max_detections_idx] + else: + # Dimension of recall: [TxKxAxM] + s = self.results["recall"] + if iou_thr is not None: + t = np.where(iou_thr == self.params.iou_thrs)[0] + s = s[t] + s = s[:, :, area_range_idx, max_detections_idx] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iou_str, area_range, max_dets, mean_s)) + return mean_s + + def _summarize_predictions(): + stats = np.zeros((12,)) + stats[0] = _summarize(use_ap=True) + stats[1] = _summarize( + use_ap=True, iou_thr=0.5, max_dets=self.params.max_dets[2] + ) + stats[2] = _summarize( + use_ap=True, iou_thr=0.75, max_dets=self.params.max_dets[2] + ) + stats[3] = _summarize( + use_ap=True, + area_range=ObjectSize.SMALL, + max_dets=self.params.max_dets[2], + ) + stats[4] = _summarize( + use_ap=True, + area_range=ObjectSize.MEDIUM, + max_dets=self.params.max_dets[2], + ) + stats[5] = _summarize( + use_ap=True, + area_range=ObjectSize.LARGE, + max_dets=self.params.max_dets[2], + ) + stats[6] = _summarize(use_ap=False, max_dets=self.params.max_dets[0]) + stats[7] = _summarize(use_ap=False, max_dets=self.params.max_dets[1]) + stats[8] = _summarize(use_ap=False, max_dets=self.params.max_dets[2]) + stats[9] = _summarize( + use_ap=False, + area_range=ObjectSize.SMALL, + max_dets=self.params.max_dets[2], + ) + stats[10] = _summarize( + use_ap=False, + area_range=ObjectSize.MEDIUM, + max_dets=self.params.max_dets[2], + ) + stats[11] = _summarize( + use_ap=False, + area_range=ObjectSize.LARGE, + max_dets=self.params.max_dets[2], + ) + return stats + + if len(self.results) != 0: + self.stats = _summarize_predictions() + + def evaluate(self): + """ + Start the per image evaluation on all images and keeep results in + self.eval_imgs (a list of dictionaries). + """ + # Select all parameters to evaluate + self.params.img_ids = list(np.unique(self.params.img_ids)) + self.params.cat_ids = list(np.unique(self.params.cat_ids)) + self.params.max_dets = sorted(self.params.max_dets) + + self._prepare_targets_and_predictions() + + # Compute IOUs between all targets and predictions for all images and categories + self.ious = { + (img_id, cat_id): self._compute_iou(img_id, cat_id) + for img_id in self.params.img_ids + for cat_id in self.params.cat_ids + } + + # Select the largest max area (the last one: 100) + max_det = self.params.max_dets[-1] + + # Evaluate each image with all categories, area range and max detections + self.eval_imgs = [ + self._evaluate_image(img_id, cat_id, area_range, max_det) + for cat_id in self.params.cat_ids + for area_range in self.params.area_range + for img_id in self.params.img_ids + ] + + # Accumulate results + self._accumulate() + + # Uncomment to see results in pycocotools presentation format: + # self._pycocotools_summarize() + + +class MeanAveragePrecision(Metric): + """ + Mean Average Precision (mAP) is a metric used to evaluate object detection models. + It is the average of the precision-recall curves at different IoU thresholds. + + Example: + ```python + import supervision as sv + from supervision.metrics import MeanAveragePrecision + + predictions = sv.Detections(...) + targets = sv.Detections(...) + + map_metric = MeanAveragePrecision() + map_result = map_metric.update(predictions, targets).compute() + + print(map_result.map50_95) + # 0.4674 + + print(map_result) + # MeanAveragePrecisionResult: + # Metric target: MetricTarget.BOXES + # Class agnostic: False + # mAP @ 50:95: 0.4674 + # mAP @ 50: 0.5048 + # mAP @ 75: 0.4796 + # mAP scores: [0.50485 0.50377 0.50377 ...] + # IoU thresh: [0.5 0.55 0.6 ...] + # AP per class: + # 0: [0.67699 0.67699 0.67699 ...] + # ... + # Small objects: ... + # Medium objects: ... + # Large objects: ... + + map_result.plot() + ``` + + ![example_plot](\ + https://media.roboflow.com/supervision-docs/metrics/mAP_plot_example.png\ + ){ align=center width="800" } + """ + + def __init__( + self, + metric_target: MetricTarget = MetricTarget.BOXES, + class_agnostic: bool = False, + class_mapping: Optional[Dict[int, int]] = None, + image_indices: Optional[List[int]] = None, + ): + """ + Initialize the Mean Average Precision metric. + + Args: + metric_target (MetricTarget): The type of detection data to use. + class_agnostic (bool): Whether to treat all data as a single class. + """ + self._metric_target = metric_target + self._class_agnostic = class_agnostic + + self._predictions_list: List[Detections] = [] + self._targets_list: List[Detections] = [] + self._class_mapping = class_mapping + self._image_indices = image_indices + + def reset(self) -> None: + """ + Reset the metric to its initial state, clearing all stored data. + """ + self._predictions_list = [] + self._targets_list = [] + + def update( + self, + predictions: Union[Detections, List[Detections]], + targets: Union[Detections, List[Detections]], + ) -> MeanAveragePrecision: + """ + Add new predictions and targets to the metric, but do not compute the result. + + Args: + predictions (Union[Detections, List[Detections]]): The predicted detections. + targets (Union[Detections, List[Detections]]): The ground-truth detections. + + Returns: + (MeanAveragePrecision): The updated metric instance. + """ + if not isinstance(predictions, list): + predictions = [predictions] + if not isinstance(targets, list): + targets = [targets] + + if len(predictions) != len(targets): + raise ValueError( + f"The number of predictions ({len(predictions)}) and" + f" targets ({len(targets)}) during the update must be the same." + ) + + if self._class_agnostic: + predictions = deepcopy(predictions) + targets = deepcopy(targets) + + for prediction in predictions: + prediction.class_id[:] = -1 + for target in targets: + target.class_id[:] = -1 + + self._predictions_list.extend(predictions) + self._targets_list.extend(targets) + + return self + + def _compute_polygon_area(self, coords: List[float]) -> float: + """ + Computes the area of a polygon using the Shoelace formula. + + Args: + coords (list of float): Flat list of x, y coordinates, e.g., [x1, y1, x2, y2, ..., xn, yn] + + Returns: + float: Area of the polygon. + """ + if len(coords) < 6: + raise ValueError("Polygon must have at least 3 points (6 coordinates)") + + x = coords[0::2] + y = coords[1::2] + + n = len(x) + area = 0.0 + for i in range(n): + j = (i + 1) % n + area += x[i] * y[j] + area -= y[i] * x[j] + + return abs(area) / 2.0 + + def _prepare_targets(self, targets): + """Transform targets into a dictionary that can be used by the COCO evaluator.""" + images = [{"id": img_id} for img_id in range(len(targets))] + if self._image_indices is not None: + images = [ + {"id": self._image_indices[img_id.get("id")]} for img_id in images + ] + # Annotations list + annotations = [] + for image_id, image_targets in enumerate(targets): + if self._image_indices is not None: + image_id = self._image_indices[image_id] + for target in image_targets: + xyxy = target[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2] + xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] + # Get "area" and "iscrowd" from data + data = target[5] + if self._class_mapping is not None: + category_id = self._class_mapping[target[3].item()] + else: + category_id = target[3].item() + dict_annotation = { + "area": data.get("area"), + "iscrowd": data.get("iscrowd"), + "image_id": image_id, + "bbox": xywh, + "category_id": category_id, + "id": len(annotations), # incrementally increase the id + } + annotations.append(dict_annotation) + # Category list + all_cat_ids = set([annotation.get("category_id") for annotation in annotations]) + categories = [{"id": cat_id} for cat_id in all_cat_ids] + # Create coco dictionary + return { + "images": images, + "annotations": annotations, + "categories": categories, + } + + def _prepare_predictions(self, predictions): + """Transform predictions into a list of predictions that can be used by the COCO evaluator.""" + coco_predictions = [] + for image_id, image_predictions in enumerate(predictions): + if self._image_indices is not None: + image_id = self._image_indices[image_id] + for prediction in image_predictions: + xyxy = prediction[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2] + xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] + if self._class_mapping is not None: + category_id = self._class_mapping[prediction[3].item()] + else: + category_id = prediction[3].item() + dict_prediction = { + "image_id": image_id, + "bbox": xywh, + "score": prediction[2].item(), + "category_id": category_id, + } + coco_predictions.append(dict_prediction) + return coco_predictions + + def compute(self) -> MeanAveragePrecisionResult: + """ + Calculate Mean Average Precision based on predicted and ground-truth + detections at different thresholds using the COCO evaluation metrics. + Source: https://github.com/rafaelpadilla/review_object_detection_metrics + + Returns: + (MeanAveragePrecisionResult): The Mean Average Precision result. + """ + total_images_predictions = len(self._predictions_list) + total_images_targets = len(self._targets_list) + + if total_images_predictions != total_images_targets: + raise ValueError( + f"The number of predictions ({total_images_predictions}) and" + f" targets ({total_images_targets}) during the evaluation must be" + " the same." + ) + dict_targets = self._prepare_targets(self._targets_list) + lst_predictions = self._prepare_predictions(self._predictions_list) + # Create a coco object with the targets + coco_gt = EvaluationDataset(targets=dict_targets) + # Include the predictions to coco object + coco_det = coco_gt.load_predictions(lst_predictions) + # Create a coco evaluator with the predictions + cocoEval = COCOEvaluator(coco_gt, coco_det) + + # Evaluate on all images + cocoEval.evaluate() + + # Create MeanAveragePrecisionResult object for small objects + mAP_small = MeanAveragePrecisionResult( + metric_target=self._metric_target, + is_class_agnostic=self._class_agnostic, + mAP_scores=cocoEval.results["mAP_scores_small"], + ap_per_class=cocoEval.results["ap_per_class_small"], + iou_thresholds=cocoEval.params.iou_thrs, + matched_classes=cocoEval.params.cat_ids, + ) + # Create MeanAveragePrecisionResult object for medium objects + mAP_medium = MeanAveragePrecisionResult( + metric_target=self._metric_target, + is_class_agnostic=self._class_agnostic, + mAP_scores=cocoEval.results["mAP_scores_medium"], + ap_per_class=cocoEval.results["ap_per_class_medium"], + iou_thresholds=cocoEval.params.iou_thrs, + matched_classes=cocoEval.params.cat_ids, + ) + # Create MeanAveragePrecisionResult object for large objects + mAP_large = MeanAveragePrecisionResult( + metric_target=self._metric_target, + is_class_agnostic=self._class_agnostic, + mAP_scores=cocoEval.results["mAP_scores_large"], + ap_per_class=cocoEval.results["ap_per_class_large"], + iou_thresholds=cocoEval.params.iou_thrs, + matched_classes=cocoEval.params.cat_ids, + ) + + # Create the final MeanAveragePrecisionResult object + mAP_result = MeanAveragePrecisionResult( + metric_target=self._metric_target, + is_class_agnostic=self._class_agnostic, + mAP_scores=cocoEval.results["mAP_scores_all_sizes"], + ap_per_class=cocoEval.results["ap_per_class_all_sizes"], + iou_thresholds=cocoEval.params.iou_thrs, + matched_classes=cocoEval.params.cat_ids, + small_objects=mAP_small, + medium_objects=mAP_medium, + large_objects=mAP_large, + ) + return mAP_result From 36a21717eb7b630fb45f5733c10297772975d6db Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Mon, 28 Apr 2025 22:11:32 +0000 Subject: [PATCH 03/24] pre-commit (lint) fixes --- supervision/metrics/mean_average_precision.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 17f4f394..3b5eaba8 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -241,9 +241,12 @@ class MeanAveragePrecisionResult: f"maxDets=100 ] = {self.map50:.3f}\n" f" Average Precision (AP) @[ IoU=0.75 | area= all | " f"maxDets=100 ] = {self.map75:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = {self.small_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = {self.medium_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = {self.large_objects.map50_95:.3f}" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] \ + = {self.small_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \ + = {self.medium_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \ + = {self.large_objects.map50_95:.3f}" ) @@ -317,8 +320,8 @@ class EvaluationDataset: Args: img_ids (list): ids of the images that we want to retrieve. cat_ids (list): ids of the categories that we want to retrieve. - area_range (tuple): area range of the annotations that we want to retrieve in - the format [min_area, max_area]. + area_range (tuple): area range of the annotations that we want to retrieve + in the format [min_area, max_area]. iscrowd (bool): if annotations to retrieve are `iscrowded=1`. """ # If there are no filters, we use all annotations @@ -910,7 +913,7 @@ class COCOEvaluator: n for n, i in enumerate(self.params.img_ids) if i in set_image_ids ] - # Evaluting at all categories, area ranges, max number of detections, and + # Evaluating at all categories, area ranges, max number of detections, and # IoU thresholds # Loop through categories @@ -993,7 +996,7 @@ class COCOEvaluator: inds = np.searchsorted(rc, self.params.rec_thrs, side="left") for ri, pos_idx in enumerate(inds): - # Ensure pi is within the range of both arrays before using it + # Ensure pi is within the range of both arrays if 0 <= pos_idx < len(pr) and 0 <= pos_idx < len( dt_scores_sorted ): @@ -1315,7 +1318,8 @@ class MeanAveragePrecision(Metric): Computes the area of a polygon using the Shoelace formula. Args: - coords (list of float): Flat list of x, y coordinates, e.g., [x1, y1, x2, y2, ..., xn, yn] + coords (list of float): Flat list of x, y coordinates, e.g., [x1, y1, x2, + y2, ..., xn, yn] Returns: float: Area of the polygon. @@ -1336,7 +1340,7 @@ class MeanAveragePrecision(Metric): return abs(area) / 2.0 def _prepare_targets(self, targets): - """Transform targets into a dictionary that can be used by the COCO evaluator.""" + """Transform targets into a dictionary that can be used by the COCO evaluator""" images = [{"id": img_id} for img_id in range(len(targets))] if self._image_indices is not None: images = [ @@ -1376,7 +1380,8 @@ class MeanAveragePrecision(Metric): } def _prepare_predictions(self, predictions): - """Transform predictions into a list of predictions that can be used by the COCO evaluator.""" + """Transform predictions into a list of predictions that can be used by the COCO + evaluator.""" coco_predictions = [] for image_id, image_predictions in enumerate(predictions): if self._image_indices is not None: From 157ed3af06e2a4cfb389f1e6cc15682f649d55ed Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Mon, 28 Apr 2025 23:08:37 +0000 Subject: [PATCH 04/24] fix data default value --- supervision/dataset/formats/coco.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index 10f46d44..e6cf3c1b 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -118,9 +118,12 @@ def coco_annotations_to_detections( else: area = None - data = dict( - iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float) - ) + if use_iscrowd or use_precomputed_area: + data = dict( + iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float) + ) + else: + data = dict() if with_masks: mask = coco_annotations_to_masks( From 15cd4451085e1b6f719d991945739bd843fa24a8 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Mon, 28 Apr 2025 23:28:26 +0000 Subject: [PATCH 05/24] Removing unnecessary comments and including references. --- supervision/metrics/mean_average_precision.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 3b5eaba8..ef6f3c71 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -252,7 +252,10 @@ class MeanAveragePrecisionResult: class EvaluationDataset: """ - Class used to evaluate models with Mean Average Precision. + Class used representing a dataset in the right format needed by the + `COCOEvaluator` class. + + Reference: https://github.com/rafaelpadilla/review_object_detection_metrics """ def __init__(self, targets: Optional[Dict[str, Any]] = None): @@ -619,11 +622,16 @@ class COCOEvaluatorParameters: class COCOEvaluator: + """ + Evaluator class to compute COCO metrics. + """ + def __init__( self, coco_targets: EvaluationDataset, coco_predictions: EvaluationDataset ): """ Constructor of COCOEvaluator object. + Args: coco_targets (EvaluationDataset): The dataset with the ground truths. coco_predictions (EvaluationDataset): The dataset with the predictions. @@ -1104,7 +1112,8 @@ class COCOEvaluator: s = s[t] s = s[:, :, :, area_range_idx, max_detections_idx] else: - # Dimension of recall: [TxKxAxM] + # Dimension of recall: + # threshold x classes x areas x max detections s = self.results["recall"] if iou_thr is not None: t = np.where(iou_thr == self.params.iou_thrs)[0] @@ -1183,7 +1192,7 @@ class COCOEvaluator: for cat_id in self.params.cat_ids } - # Select the largest max area (the last one: 100) + # Select the largest max area (the last element containing 100 dets max_det = self.params.max_dets[-1] # Evaluate each image with all categories, area range and max detections @@ -1197,9 +1206,6 @@ class COCOEvaluator: # Accumulate results self._accumulate() - # Uncomment to see results in pycocotools presentation format: - # self._pycocotools_summarize() - class MeanAveragePrecision(Metric): """ From 688d9877652138ff6d5b03e429e646c3ee55a829 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Wed, 7 May 2025 20:28:39 +0000 Subject: [PATCH 06/24] replacing `__str__` implementation with `to_pycocotools_output` --- supervision/metrics/mean_average_precision.py | 87 +++++-------------- 1 file changed, 23 insertions(+), 64 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index ef6f3c71..124d5a9a 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -76,56 +76,34 @@ class MeanAveragePrecisionResult: def __str__(self) -> str: """ - Format as a pretty string. + Formats the evaluation output metrics to match the structure used by pycocotools Example: - ```python - print(map_result) - # MeanAveragePrecisionResult: - # Metric target: MetricTarget.BOXES - # Class agnostic: False - # mAP @ 50:95: 0.4674 - # mAP @ 50: 0.5048 - # mAP @ 75: 0.4796 - # mAP scores: [0.50485 0.50377 0.50377 ...] - # IoU thresh: [0.5 0.55 0.6 ...] - # AP per class: - # 0: [0.67699 0.67699 0.67699 ...] - # ... - # Small objects: ... - # Medium objects: ... - # Large objects: ... + ```python + print(map_result) + # MeanAveragePrecisionResult: + Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.464 + Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.637 + Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.203 + Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.284 + Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.497 + Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.629 ``` """ - - out_str = ( - f"{self.__class__.__name__}:\n" - f"Metric target: {self.metric_target}\n" - f"Class agnostic: {self.is_class_agnostic}\n" - f"mAP @ 50:95: {self.map50_95:.4f}\n" - f"mAP @ 50: {self.map50:.4f}\n" - f"mAP @ 75: {self.map75:.4f}\n" - f"mAP scores: {self.mAP_scores}\n" - f"IoU thresh: {self.iou_thresholds}\n" - f"AP per class:\n" + return ( + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " + f"maxDets=100 ] = {self.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50 | area= all | " + f"maxDets=100 ] = {self.map50:.3f}\n" + f" Average Precision (AP) @[ IoU=0.75 | area= all | " + f"maxDets=100 ] = {self.map75:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] \ + = {self.small_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \ + = {self.medium_objects.map50_95:.3f}\n" + f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \ + = {self.large_objects.map50_95:.3f}" ) - if self.ap_per_class.size == 0: - out_str += " No results\n" - for class_id, ap_of_class in zip(self.matched_classes, self.ap_per_class): - out_str += f" {class_id}: {ap_of_class}\n" - - indent = " " - if self.small_objects is not None: - indented = indent + str(self.small_objects).replace("\n", f"\n{indent}") - out_str += f"\nSmall objects:\n{indented}" - if self.medium_objects is not None: - indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}") - out_str += f"\nMedium objects:\n{indented}" - if self.large_objects is not None: - indented = indent + str(self.large_objects).replace("\n", f"\n{indent}") - out_str += f"\nLarge objects:\n{indented}" - - return out_str def to_pandas(self) -> "pd.DataFrame": """ @@ -230,25 +208,6 @@ class MeanAveragePrecisionResult: plt.tight_layout() plt.show() - def to_pycocotools_output(self) -> str: - """ - Convert the result to the same output format as pycocotools. - """ - return ( - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " - f"maxDets=100 ] = {self.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50 | area= all | " - f"maxDets=100 ] = {self.map50:.3f}\n" - f" Average Precision (AP) @[ IoU=0.75 | area= all | " - f"maxDets=100 ] = {self.map75:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] \ - = {self.small_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \ - = {self.medium_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \ - = {self.large_objects.map50_95:.3f}" - ) - class EvaluationDataset: """ From bba43ecdf43c6464e2dd327ea4fa962008063789 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Wed, 7 May 2025 20:31:05 +0000 Subject: [PATCH 07/24] Updating docstring to reflect newly added arguments --- supervision/metrics/mean_average_precision.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 124d5a9a..d0294526 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -1222,6 +1222,9 @@ class MeanAveragePrecision(Metric): Args: metric_target (MetricTarget): The type of detection data to use. class_agnostic (bool): Whether to treat all data as a single class. + class_mapping (Optional[Dict[int, int]]): A dictionary to map class IDs to + new IDs. + image_indices (Optional[List[int]]): The indices of the images to use. """ self._metric_target = metric_target self._class_agnostic = class_agnostic From 790baa615f01e45282593ed7ca75d2f5e49c417a Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Wed, 7 May 2025 21:36:24 +0000 Subject: [PATCH 08/24] Removing spaces --- supervision/metrics/mean_average_precision.py | 44 ++++--------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index d0294526..1a7365e3 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -91,18 +91,18 @@ class MeanAveragePrecisionResult: ``` """ return ( - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " + f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | " f"maxDets=100 ] = {self.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50 | area= all | " + f"Average Precision (AP) @[ IoU=0.50 | area= all | " f"maxDets=100 ] = {self.map50:.3f}\n" - f" Average Precision (AP) @[ IoU=0.75 | area= all | " + f"Average Precision (AP) @[ IoU=0.75 | area= all | " f"maxDets=100 ] = {self.map75:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] \ - = {self.small_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \ - = {self.medium_objects.map50_95:.3f}\n" - f" Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \ - = {self.large_objects.map50_95:.3f}" + f"Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] " + f"= {self.small_objects.map50_95:.3f}\n" + f"Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] " + f"= {self.medium_objects.map50_95:.3f}\n" + f"Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] " + f"= {self.large_objects.map50_95:.3f}" ) def to_pandas(self) -> "pd.DataFrame": @@ -1281,32 +1281,6 @@ class MeanAveragePrecision(Metric): return self - def _compute_polygon_area(self, coords: List[float]) -> float: - """ - Computes the area of a polygon using the Shoelace formula. - - Args: - coords (list of float): Flat list of x, y coordinates, e.g., [x1, y1, x2, - y2, ..., xn, yn] - - Returns: - float: Area of the polygon. - """ - if len(coords) < 6: - raise ValueError("Polygon must have at least 3 points (6 coordinates)") - - x = coords[0::2] - y = coords[1::2] - - n = len(x) - area = 0.0 - for i in range(n): - j = (i + 1) % n - area += x[i] * y[j] - area -= y[i] * x[j] - - return abs(area) / 2.0 - def _prepare_targets(self, targets): """Transform targets into a dictionary that can be used by the COCO evaluator""" images = [{"id": img_id} for img_id in range(len(targets))] From 19c4e44e2ce6528682fe44dfd74c6b316b904481 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Wed, 7 May 2025 22:10:15 +0000 Subject: [PATCH 09/24] removing unnecessary flag --- supervision/dataset/core.py | 5 ----- supervision/dataset/formats/coco.py | 13 ++----------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index dc5de4d5..2fb3aa4c 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -563,7 +563,6 @@ class DetectionDataset(BaseDataset): images_directory_path: str, annotations_path: str, force_masks: bool = False, - use_precomputed_area: bool = False, use_iscrowd: bool = False, ) -> DetectionDataset: """ @@ -576,9 +575,6 @@ class DetectionDataset(BaseDataset): force_masks (bool): If True, forces masks to be loaded for all annotations, regardless of whether they are present. - use_precomputed_area (bool): If True, - uses precomputed area for all annotations, setting it to None if not - present. use_iscrowd (bool): If True, uses COCO's property `iscrowd` in all annotations, regardless of whether they are present. If not presented, `iscrowd=0` @@ -612,7 +608,6 @@ class DetectionDataset(BaseDataset): images_directory_path=images_directory_path, annotations_path=annotations_path, force_masks=force_masks, - use_precomputed_area=use_precomputed_area, use_iscrowd=use_iscrowd, ) return DetectionDataset(classes=classes, images=images, annotations=annotations) diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index e6cf3c1b..3ef86b47 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -94,7 +94,6 @@ def coco_annotations_to_detections( resolution_wh: Tuple[int, int], with_masks: bool, use_iscrowd: bool = False, - use_precomputed_area: bool = False, ) -> Detections: if not image_annotations: return Detections.empty() @@ -110,19 +109,13 @@ def coco_annotations_to_detections( iscrowd = [ image_annotation["iscrowd"] for image_annotation in image_annotations ] - else: - iscrowd = [0] * len(image_annotations) - - if use_precomputed_area: area = [image_annotation["area"] for image_annotation in image_annotations] - else: - area = None - - if use_iscrowd or use_precomputed_area: data = dict( iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float) ) else: + iscrowd = [0] * len(image_annotations) + area = None data = dict() if with_masks: @@ -197,7 +190,6 @@ def load_coco_annotations( annotations_path: str, force_masks: bool = False, use_iscrowd: bool = False, - use_precomputed_area: bool = False, ) -> Tuple[List[str], List[str], Dict[str, Detections]]: coco_data = read_json_file(file_path=annotations_path) classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) @@ -228,7 +220,6 @@ def load_coco_annotations( resolution_wh=(image_width, image_height), with_masks=force_masks, use_iscrowd=use_iscrowd, - use_precomputed_area=use_precomputed_area, ) annotation = map_detections_class_id( From 8f72e490802d2d83fd53c2fb99c3a2f29fc42689 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Thu, 8 May 2025 01:46:46 +0000 Subject: [PATCH 10/24] Moving iou_with_jaccard and jaccard from mean_average_precision.py to detection/utils.py --- supervision/detection/utils.py | 70 +++++++++++++++++ supervision/metrics/mean_average_precision.py | 78 ++----------------- 2 files changed, 76 insertions(+), 72 deletions(-) diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 8153628b..485c0e80 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -1325,3 +1325,73 @@ def spread_out_boxes( xyxy_padded[:, [2, 3]] += force_vectors return pad_boxes(xyxy_padded, px=-1) + + +def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: + """ + Calculate the Jaccard index (intersection over union) between two bounding boxes. + If a gt object is marked as "iscrowd", a dt is allowed to match any subregion + of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using + gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing + iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) + + Args: + box_a (List[float]): Box coordinates in the format [x, y, width, height]. + box_b (List[float]): Box coordinates in the format [x, y, width, height]. + iscrowd (bool): Flag indicating if the second box is a crowd region or not. + + Returns: + float: Jaccard index between the two bounding boxes. + """ + # Smallest number to avoid division by zero + EPS = np.spacing(1) + + xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] + xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] + + # Innermost left x + xi = max(xa, xb) + # Innermost right x + x2i = min(x2a, x2b) + # Same for y + yi = max(ya, yb) + y2i = min(y2a, y2b) + + # Calculate areas + Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) + Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) + Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) + + if is_crowd: + return Ai / (Aa + EPS) + + return Ai / (Aa + Ab - Ai + EPS) + + +def iou_with_jaccard( + dt: List[List[float]], gt: List[List[float]], is_crowd: List[bool] +) -> np.ndarray: + """ + Calculate the intersection over union (IoU) between detection bounding boxes (dt) + and ground-truth bounding boxes (gt). + Reference: https://github.com/rafaelpadilla/review_object_detection_metrics + + Args: + dt (List[List[float]]): List of detection bounding boxes in the \ + format [x, y, width, height]. + gt (List[List[float]]): List of ground-truth bounding boxes in the \ + format [x, y, width, height]. + is_crowd (List[bool]): List indicating if each ground-truth bounding box \ + is a crowd region or not. + + Returns: + np.ndarray: Array of IoU values of shape (len(dt), len(gt)). + """ + assert len(is_crowd) == len(gt), "iou(iscrowd=) must have the same length as gt" + if len(dt) == 0 or len(gt) == 0: + return np.array([]) + ious = np.zeros((len(dt), len(gt)), dtype=np.float64) + for g_idx, g in enumerate(gt): + for d_idx, d in enumerate(dt): + ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) + return ious diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 1a7365e3..2d58467f 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -16,6 +16,7 @@ from supervision.detection.core import Detections from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget from supervision.metrics.utils.utils import ensure_pandas_installed +from supervision.detection.utils import iou_with_jaccard if TYPE_CHECKING: import pandas as pd @@ -99,10 +100,10 @@ class MeanAveragePrecisionResult: f"maxDets=100 ] = {self.map75:.3f}\n" f"Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] " f"= {self.small_objects.map50_95:.3f}\n" - f"Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] " - f"= {self.medium_objects.map50_95:.3f}\n" - f"Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] " - f"= {self.large_objects.map50_95:.3f}" + f"Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \ + = {self.medium_objects.map50_95:.3f}\n" + f"Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \ + = {self.large_objects.map50_95:.3f}" ) def to_pandas(self) -> "pd.DataFrame": @@ -485,73 +486,6 @@ class ObjectSize(Enum): LARGE = "large" -def _iou_with_jaccard( - dt: List[List[float]], gt: List[List[float]], is_crowd: List[bool] -) -> np.ndarray: - """ - Calculate the intersection over union (IoU) between detection bounding boxes (dt) - and ground-truth bounding boxes (gt). - Reference: https://github.com/rafaelpadilla/review_object_detection_metrics - - Args: - dt (List[List[float]]): List of detection bounding boxes in the \ - format [x, y, width, height]. - gt (List[List[float]]): List of ground-truth bounding boxes in the \ - format [x, y, width, height]. - is_crowd (List[bool]): List indicating if each ground-truth bounding box \ - is a crowd region or not. - - Returns: - np.ndarray: Array of IoU values of shape (len(dt), len(gt)). - """ - assert len(is_crowd) == len(gt), "iou(iscrowd=) must have the same length as gt" - if len(dt) == 0 or len(gt) == 0: - return np.array([]) - ious = np.zeros((len(dt), len(gt)), dtype=np.float64) - for g_idx, g in enumerate(gt): - for d_idx, d in enumerate(dt): - ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) - return ious - - -def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: - """ - Calculate the Jaccard index (intersection over union) between two bounding boxes. - If a gt object is marked as "iscrowd", a dt is allowed to match any subregion - of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using - gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing - iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) - - Args: - box_a (List[float]): Box coordinates in the format [x, y, width, height]. - box_b (List[float]): Box coordinates in the format [x, y, width, height]. - iscrowd (bool): Flag indicating if the second box is a crowd region or not. - - Returns: - float: Jaccard index between the two bounding boxes. - """ - xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] - xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] - - # Innermost left x - xi = max(xa, xb) - # Innermost right x - x2i = min(x2a, x2b) - # Same for y - yi = max(ya, yb) - y2i = min(y2a, y2b) - - # Calculate areas - Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) - Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) - Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) - - if is_crowd: - return Ai / (Aa + EPS) - - return Ai / (Aa + Ab - Ai + EPS) - - class COCOEvaluatorParameters: """ Parameters for COCOEvaluator @@ -689,7 +623,7 @@ class COCOEvaluator: # Get the iscrowd flag for each gt is_crowd = [int(o["iscrowd"]) for o in gt] # Compute iou between each prediction a and gt region - iou = _iou_with_jaccard(dt_boxes, gt_boxes, is_crowd) + iou = iou_with_jaccard(dt_boxes, gt_boxes, is_crowd) return iou def _evaluate_image( From 52980a6aa7e6c460fc9e8104db0ce9dbeb33da85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 01:52:42 +0000 Subject: [PATCH 11/24] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/metrics/mean_average_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 2d58467f..3b963795 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -13,10 +13,10 @@ import numpy as np from matplotlib import pyplot as plt from supervision.detection.core import Detections +from supervision.detection.utils import iou_with_jaccard from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget from supervision.metrics.utils.utils import ensure_pandas_installed -from supervision.detection.utils import iou_with_jaccard if TYPE_CHECKING: import pandas as pd From cff974555e9d5f2245d9fa83940abcc114a84d3f Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 1 Jun 2025 15:52:01 +0000 Subject: [PATCH 12/24] 1) Simplifying the logic of `use_iscrowd` and compatibility with other datasets. 2) Making `get_coco_class_index_mapping` return the inversed mapping, to simplify its usage. --- supervision/dataset/core.py | 6 ------ supervision/dataset/formats/coco.py | 12 +++++------- supervision/metrics/mean_average_precision.py | 10 ++++------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index 2fb3aa4c..c6fc760e 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -563,7 +563,6 @@ class DetectionDataset(BaseDataset): images_directory_path: str, annotations_path: str, force_masks: bool = False, - use_iscrowd: bool = False, ) -> DetectionDataset: """ Creates a Dataset instance from COCO formatted data. @@ -575,10 +574,6 @@ class DetectionDataset(BaseDataset): force_masks (bool): If True, forces masks to be loaded for all annotations, regardless of whether they are present. - use_iscrowd (bool): If True, - uses COCO's property `iscrowd` in all annotations, - regardless of whether they are present. If not presented, `iscrowd=0` - will be used. Returns: DetectionDataset: A DetectionDataset instance containing the loaded images and annotations. @@ -608,7 +603,6 @@ class DetectionDataset(BaseDataset): images_directory_path=images_directory_path, annotations_path=annotations_path, force_masks=force_masks, - use_iscrowd=use_iscrowd, ) return DetectionDataset(classes=classes, images=images, annotations=annotations) diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index 3ef86b47..27b17243 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -105,6 +105,7 @@ def coco_annotations_to_detections( xyxy = np.asarray(xyxy) xyxy[:, 2:4] += xyxy[:, 0:2] + data = dict() if use_iscrowd: iscrowd = [ image_annotation["iscrowd"] for image_annotation in image_annotations @@ -113,10 +114,6 @@ def coco_annotations_to_detections( data = dict( iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float) ) - else: - iscrowd = [0] * len(image_annotations) - area = None - data = dict() if with_masks: mask = coco_annotations_to_masks( @@ -182,14 +179,15 @@ def get_coco_class_index_mapping(annotations_path: str) -> Dict[int, int]: class_mapping = build_coco_class_index_mapping( coco_categories=coco_data["categories"], target_classes=classes ) - return class_mapping + inv_class_mapping = {v: k for k, v in class_mapping.items()} + return inv_class_mapping def load_coco_annotations( images_directory_path: str, annotations_path: str, force_masks: bool = False, - use_iscrowd: bool = False, + # use_iscrowd: bool = True, ) -> Tuple[List[str], List[str], Dict[str, Detections]]: coco_data = read_json_file(file_path=annotations_path) classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) @@ -219,7 +217,7 @@ def load_coco_annotations( image_annotations=image_annotations, resolution_wh=(image_width, image_height), with_masks=force_masks, - use_iscrowd=use_iscrowd, + use_iscrowd=True, ) annotation = map_detections_class_id( diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 3b963795..ac7351a9 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -666,9 +666,6 @@ class COCOEvaluator: dt_sorted = np.argsort([-d["score"] for d in dt], kind="stable") dt = [dt[i] for i in dt_sorted[0:max_det]] - # Get the iscrowd flag for each gt - # iscrowd = [int(o["iscrowd"]) for o in gt] - # Load computed ious for the given image and category ious = ( self.ious[img_id, cat_id][:, gt_sorted] @@ -1230,15 +1227,16 @@ class MeanAveragePrecision(Metric): for target in image_targets: xyxy = target[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2] xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] - # Get "area" and "iscrowd" from data + # Get "area" and "iscrowd" (default 0) from data data = target[5] + if self._class_mapping is not None: category_id = self._class_mapping[target[3].item()] else: category_id = target[3].item() dict_annotation = { - "area": data.get("area"), - "iscrowd": data.get("iscrowd"), + "area": data.get("area", 0), + "iscrowd": data.get("iscrowd", 0), "image_id": image_id, "bbox": xywh, "category_id": category_id, From fd3135145aeb0631bf4dbe4ccc497165304e171e Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 1 Jun 2025 17:21:45 +0000 Subject: [PATCH 13/24] renaming and reordering inputs of function to keep consistency with other functions --- supervision/detection/utils.py | 20 +++++++++++-------- supervision/metrics/mean_average_precision.py | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 485c0e80..7ce199e5 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -1369,7 +1369,9 @@ def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: def iou_with_jaccard( - dt: List[List[float]], gt: List[List[float]], is_crowd: List[bool] + boxes_true: List[List[float]], + boxes_detection: List[List[float]], + is_crowd: List[bool], ) -> np.ndarray: """ Calculate the intersection over union (IoU) between detection bounding boxes (dt) @@ -1377,9 +1379,9 @@ def iou_with_jaccard( Reference: https://github.com/rafaelpadilla/review_object_detection_metrics Args: - dt (List[List[float]]): List of detection bounding boxes in the \ + boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \ format [x, y, width, height]. - gt (List[List[float]]): List of ground-truth bounding boxes in the \ + boxes_detection (List[List[float]]): List of detection bounding boxes in the \ format [x, y, width, height]. is_crowd (List[bool]): List indicating if each ground-truth bounding box \ is a crowd region or not. @@ -1387,11 +1389,13 @@ def iou_with_jaccard( Returns: np.ndarray: Array of IoU values of shape (len(dt), len(gt)). """ - assert len(is_crowd) == len(gt), "iou(iscrowd=) must have the same length as gt" - if len(dt) == 0 or len(gt) == 0: + assert len(is_crowd) == len(boxes_true), ( + "iou(iscrowd=) must have the same length as boxes_true" + ) + if len(boxes_detection) == 0 or len(boxes_true) == 0: return np.array([]) - ious = np.zeros((len(dt), len(gt)), dtype=np.float64) - for g_idx, g in enumerate(gt): - for d_idx, d in enumerate(dt): + ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64) + for g_idx, g in enumerate(boxes_true): + for d_idx, d in enumerate(boxes_detection): ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) return ious diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index ac7351a9..bb2ef8a2 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -623,7 +623,7 @@ class COCOEvaluator: # Get the iscrowd flag for each gt is_crowd = [int(o["iscrowd"]) for o in gt] # Compute iou between each prediction a and gt region - iou = iou_with_jaccard(dt_boxes, gt_boxes, is_crowd) + iou = iou_with_jaccard(gt_boxes, dt_boxes, is_crowd) return iou def _evaluate_image( From 5f9c55d9f37e1d0b660f566c796359068d08a131 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Tue, 1 Jul 2025 23:16:27 +0000 Subject: [PATCH 14/24] keeping argument use_iscrowd in function load_coco_annotations(...) ; moving box_iou_batch_with_jaccard(...) to supervision/__init__.py --- supervision/__init__.py | 104 ++++++++++++++++++++++++++++ supervision/dataset/formats/coco.py | 4 +- 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/supervision/__init__.py b/supervision/__init__.py index de2818fa..7141d936 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -1,4 +1,7 @@ import importlib.metadata as importlib_metadata +from typing import List + +import numpy as np try: # This will read version from pyproject.toml @@ -180,6 +183,7 @@ __all__ = [ "VideoInfo", "VideoSink", "box_iou_batch", + "box_iou_batch_with_jaccard", "box_non_max_merge", "box_non_max_suppression", "calculate_masks_centroids", @@ -229,3 +233,103 @@ __all__ = [ "xyxy_to_xyah", "xyxy_to_xywh", ] + + +def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: + """ + Calculate the Jaccard index (intersection over union) between two bounding boxes. + If a gt object is marked as "iscrowd", a dt is allowed to match any subregion + of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using + gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing + iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) + + Args: + box_a (List[float]): Box coordinates in the format [x, y, width, height]. + box_b (List[float]): Box coordinates in the format [x, y, width, height]. + iscrowd (bool): Flag indicating if the second box is a crowd region or not. + + Returns: + float: Jaccard index between the two bounding boxes. + """ + # Smallest number to avoid division by zero + EPS = np.spacing(1) + + xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] + xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] + + # Innermost left x + xi = max(xa, xb) + # Innermost right x + x2i = min(x2a, x2b) + # Same for y + yi = max(ya, yb) + y2i = min(y2a, y2b) + + # Calculate areas + Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) + Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) + Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) + + if is_crowd: + return Ai / (Aa + EPS) + + return Ai / (Aa + Ab - Ai + EPS) + + +def box_iou_batch_with_jaccard( + boxes_true: List[List[float]], + boxes_detection: List[List[float]], + is_crowd: List[bool], +) -> np.ndarray: + """ + Calculate the intersection over union (IoU) between detection bounding boxes (dt) + and ground-truth bounding boxes (gt). + Reference: https://github.com/rafaelpadilla/review_object_detection_metrics + + Args: + boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \ + format [x, y, width, height]. + boxes_detection (List[List[float]]): List of detection bounding boxes in the \ + format [x, y, width, height]. + is_crowd (List[bool]): List indicating if each ground-truth bounding box \ + is a crowd region or not. + + Returns: + np.ndarray: Array of IoU values of shape (len(dt), len(gt)). + + Examples: + ```python + import numpy as np + import supervision as sv + + boxes_true = [ + [10, 20, 30, 40], # x, y, w, h + [15, 25, 35, 45] + ] + boxes_detection = [ + [12, 22, 28, 38], + [16, 26, 36, 46] + ] + is_crowd = [False, False] + + ious = sv.box_iou_batch_with_jaccard( + boxes_true=boxes_true, + boxes_detection=boxes_detection, + is_crowd=is_crowd + ) + # array([ + # [0.8866..., 0.4960...], + # [0.4000..., 0.8622...] + # ]) + ``` + """ + assert len(is_crowd) == len(boxes_true), ( + "iou(iscrowd=) must have the same length as boxes_true" + ) + if len(boxes_detection) == 0 or len(boxes_true) == 0: + return np.array([]) + ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64) + for g_idx, g in enumerate(boxes_true): + for d_idx, d in enumerate(boxes_detection): + ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) + return ious diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index 27b17243..ac1b7961 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -187,7 +187,7 @@ def load_coco_annotations( images_directory_path: str, annotations_path: str, force_masks: bool = False, - # use_iscrowd: bool = True, + use_iscrowd: bool = True, ) -> Tuple[List[str], List[str], Dict[str, Detections]]: coco_data = read_json_file(file_path=annotations_path) classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) @@ -217,7 +217,7 @@ def load_coco_annotations( image_annotations=image_annotations, resolution_wh=(image_width, image_height), with_masks=force_masks, - use_iscrowd=True, + use_iscrowd=use_iscrowd, ) annotation = map_detections_class_id( From 0a55b3b3a238f32da44b72980c2c0b0c9b24ce01 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Tue, 1 Jul 2025 23:17:48 +0000 Subject: [PATCH 15/24] making mean_average_precision.py have a lazy import of box_iou_batch_with_jaccard(...) --- supervision/metrics/mean_average_precision.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index bb2ef8a2..12587b95 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -13,7 +13,6 @@ import numpy as np from matplotlib import pyplot as plt from supervision.detection.core import Detections -from supervision.detection.utils import iou_with_jaccard from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget from supervision.metrics.utils.utils import ensure_pandas_installed @@ -601,6 +600,8 @@ class COCOEvaluator: Returns: np.ndarray: The IoU between the targets and predictions. """ + from supervision import box_iou_batch_with_jaccard + gt = self._targets[img_id, cat_id] dt = self._predictions[img_id, cat_id] @@ -623,7 +624,7 @@ class COCOEvaluator: # Get the iscrowd flag for each gt is_crowd = [int(o["iscrowd"]) for o in gt] # Compute iou between each prediction a and gt region - iou = iou_with_jaccard(gt_boxes, dt_boxes, is_crowd) + iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd) return iou def _evaluate_image( From 49a8fd96d653b6061356fafec9cd739b64d4bbf9 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Tue, 1 Jul 2025 23:18:47 +0000 Subject: [PATCH 16/24] renaming iou_with_jaccard(...) to box_iou_batch_with_jaccard(...) and moving it to supervision.__init__.py --- supervision/detection/utils.py | 73 ---------------------------------- 1 file changed, 73 deletions(-) diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 7ce199e5..2fa6849f 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -1326,76 +1326,3 @@ def spread_out_boxes( return pad_boxes(xyxy_padded, px=-1) - -def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: - """ - Calculate the Jaccard index (intersection over union) between two bounding boxes. - If a gt object is marked as "iscrowd", a dt is allowed to match any subregion - of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using - gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing - iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) - - Args: - box_a (List[float]): Box coordinates in the format [x, y, width, height]. - box_b (List[float]): Box coordinates in the format [x, y, width, height]. - iscrowd (bool): Flag indicating if the second box is a crowd region or not. - - Returns: - float: Jaccard index between the two bounding boxes. - """ - # Smallest number to avoid division by zero - EPS = np.spacing(1) - - xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] - xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] - - # Innermost left x - xi = max(xa, xb) - # Innermost right x - x2i = min(x2a, x2b) - # Same for y - yi = max(ya, yb) - y2i = min(y2a, y2b) - - # Calculate areas - Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) - Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) - Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) - - if is_crowd: - return Ai / (Aa + EPS) - - return Ai / (Aa + Ab - Ai + EPS) - - -def iou_with_jaccard( - boxes_true: List[List[float]], - boxes_detection: List[List[float]], - is_crowd: List[bool], -) -> np.ndarray: - """ - Calculate the intersection over union (IoU) between detection bounding boxes (dt) - and ground-truth bounding boxes (gt). - Reference: https://github.com/rafaelpadilla/review_object_detection_metrics - - Args: - boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \ - format [x, y, width, height]. - boxes_detection (List[List[float]]): List of detection bounding boxes in the \ - format [x, y, width, height]. - is_crowd (List[bool]): List indicating if each ground-truth bounding box \ - is a crowd region or not. - - Returns: - np.ndarray: Array of IoU values of shape (len(dt), len(gt)). - """ - assert len(is_crowd) == len(boxes_true), ( - "iou(iscrowd=) must have the same length as boxes_true" - ) - if len(boxes_detection) == 0 or len(boxes_true) == 0: - return np.array([]) - ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64) - for g_idx, g in enumerate(boxes_true): - for d_idx, d in enumerate(boxes_detection): - ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) - return ious From 4414e15a000cca5739eecef57fc87b963d947310 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 23:19:22 +0000 Subject: [PATCH 17/24] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/detection/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 2fa6849f..8153628b 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -1325,4 +1325,3 @@ def spread_out_boxes( xyxy_padded[:, [2, 3]] += force_vectors return pad_boxes(xyxy_padded, px=-1) - From d543be924f5f09ff235ed8f61fabcfbd4d49b024 Mon Sep 17 00:00:00 2001 From: Rafael Padilla <31217453+rafaelpadilla@users.noreply.github.com> Date: Sat, 5 Jul 2025 21:41:02 -0300 Subject: [PATCH 18/24] Update supervision/dataset/formats/coco.py Co-authored-by: Piotr Skalski --- supervision/dataset/formats/coco.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index ac1b7961..a0da8693 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -179,8 +179,7 @@ def get_coco_class_index_mapping(annotations_path: str) -> Dict[int, int]: class_mapping = build_coco_class_index_mapping( coco_categories=coco_data["categories"], target_classes=classes ) - inv_class_mapping = {v: k for k, v in class_mapping.items()} - return inv_class_mapping + return {v: k for k, v in class_mapping.items()} def load_coco_annotations( From c7b4993721aade9bdd3c179f22dfbf8829582132 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 6 Jul 2025 01:31:30 +0000 Subject: [PATCH 19/24] Moving `_jaccard` and `box_iou_batch_with_jaccard` to `supervision/detection/utils.py`. --- supervision/__init__.py | 101 +----------------- supervision/detection/utils.py | 100 +++++++++++++++++ supervision/metrics/mean_average_precision.py | 2 +- 3 files changed, 102 insertions(+), 101 deletions(-) diff --git a/supervision/__init__.py b/supervision/__init__.py index 7141d936..768212a5 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -61,6 +61,7 @@ from supervision.detection.tools.polygon_zone import PolygonZone, PolygonZoneAnn from supervision.detection.tools.smoother import DetectionsSmoother from supervision.detection.utils import ( box_iou_batch, + box_iou_batch_with_jaccard, calculate_masks_centroids, clip_boxes, contains_holes, @@ -233,103 +234,3 @@ __all__ = [ "xyxy_to_xyah", "xyxy_to_xywh", ] - - -def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: - """ - Calculate the Jaccard index (intersection over union) between two bounding boxes. - If a gt object is marked as "iscrowd", a dt is allowed to match any subregion - of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using - gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing - iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) - - Args: - box_a (List[float]): Box coordinates in the format [x, y, width, height]. - box_b (List[float]): Box coordinates in the format [x, y, width, height]. - iscrowd (bool): Flag indicating if the second box is a crowd region or not. - - Returns: - float: Jaccard index between the two bounding boxes. - """ - # Smallest number to avoid division by zero - EPS = np.spacing(1) - - xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] - xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] - - # Innermost left x - xi = max(xa, xb) - # Innermost right x - x2i = min(x2a, x2b) - # Same for y - yi = max(ya, yb) - y2i = min(y2a, y2b) - - # Calculate areas - Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) - Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) - Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) - - if is_crowd: - return Ai / (Aa + EPS) - - return Ai / (Aa + Ab - Ai + EPS) - - -def box_iou_batch_with_jaccard( - boxes_true: List[List[float]], - boxes_detection: List[List[float]], - is_crowd: List[bool], -) -> np.ndarray: - """ - Calculate the intersection over union (IoU) between detection bounding boxes (dt) - and ground-truth bounding boxes (gt). - Reference: https://github.com/rafaelpadilla/review_object_detection_metrics - - Args: - boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \ - format [x, y, width, height]. - boxes_detection (List[List[float]]): List of detection bounding boxes in the \ - format [x, y, width, height]. - is_crowd (List[bool]): List indicating if each ground-truth bounding box \ - is a crowd region or not. - - Returns: - np.ndarray: Array of IoU values of shape (len(dt), len(gt)). - - Examples: - ```python - import numpy as np - import supervision as sv - - boxes_true = [ - [10, 20, 30, 40], # x, y, w, h - [15, 25, 35, 45] - ] - boxes_detection = [ - [12, 22, 28, 38], - [16, 26, 36, 46] - ] - is_crowd = [False, False] - - ious = sv.box_iou_batch_with_jaccard( - boxes_true=boxes_true, - boxes_detection=boxes_detection, - is_crowd=is_crowd - ) - # array([ - # [0.8866..., 0.4960...], - # [0.4000..., 0.8622...] - # ]) - ``` - """ - assert len(is_crowd) == len(boxes_true), ( - "iou(iscrowd=) must have the same length as boxes_true" - ) - if len(boxes_detection) == 0 or len(boxes_true) == 0: - return np.array([]) - ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64) - for g_idx, g in enumerate(boxes_true): - for d_idx, d in enumerate(boxes_detection): - ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) - return ious diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 8153628b..d86990ee 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -1325,3 +1325,103 @@ def spread_out_boxes( xyxy_padded[:, [2, 3]] += force_vectors return pad_boxes(xyxy_padded, px=-1) + + +def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float: + """ + Calculate the Jaccard index (intersection over union) between two bounding boxes. + If a gt object is marked as "iscrowd", a dt is allowed to match any subregion + of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using + gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing + iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) + + Args: + box_a (List[float]): Box coordinates in the format [x, y, width, height]. + box_b (List[float]): Box coordinates in the format [x, y, width, height]. + iscrowd (bool): Flag indicating if the second box is a crowd region or not. + + Returns: + float: Jaccard index between the two bounding boxes. + """ + # Smallest number to avoid division by zero + EPS = np.spacing(1) + + xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3] + xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3] + + # Innermost left x + xi = max(xa, xb) + # Innermost right x + x2i = min(x2a, x2b) + # Same for y + yi = max(ya, yb) + y2i = min(y2a, y2b) + + # Calculate areas + Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0) + Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0) + Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0) + + if is_crowd: + return Ai / (Aa + EPS) + + return Ai / (Aa + Ab - Ai + EPS) + + +def box_iou_batch_with_jaccard( + boxes_true: List[List[float]], + boxes_detection: List[List[float]], + is_crowd: List[bool], +) -> np.ndarray: + """ + Calculate the intersection over union (IoU) between detection bounding boxes (dt) + and ground-truth bounding boxes (gt). + Reference: https://github.com/rafaelpadilla/review_object_detection_metrics + + Args: + boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \ + format [x, y, width, height]. + boxes_detection (List[List[float]]): List of detection bounding boxes in the \ + format [x, y, width, height]. + is_crowd (List[bool]): List indicating if each ground-truth bounding box \ + is a crowd region or not. + + Returns: + np.ndarray: Array of IoU values of shape (len(dt), len(gt)). + + Examples: + ```python + import numpy as np + import supervision as sv + + boxes_true = [ + [10, 20, 30, 40], # x, y, w, h + [15, 25, 35, 45] + ] + boxes_detection = [ + [12, 22, 28, 38], + [16, 26, 36, 46] + ] + is_crowd = [False, False] + + ious = sv.box_iou_batch_with_jaccard( + boxes_true=boxes_true, + boxes_detection=boxes_detection, + is_crowd=is_crowd + ) + # array([ + # [0.8866..., 0.4960...], + # [0.4000..., 0.8622...] + # ]) + ``` + """ + assert len(is_crowd) == len(boxes_true), ( + "`is_crowd` must have the same length as `boxes_true`" + ) + if len(boxes_detection) == 0 or len(boxes_true) == 0: + return np.array([]) + ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64) + for g_idx, g in enumerate(boxes_true): + for d_idx, d in enumerate(boxes_detection): + ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx]) + return ious diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index 12587b95..8e8d18d7 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np from matplotlib import pyplot as plt +from supervision import box_iou_batch_with_jaccard from supervision.detection.core import Detections from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget @@ -600,7 +601,6 @@ class COCOEvaluator: Returns: np.ndarray: The IoU between the targets and predictions. """ - from supervision import box_iou_batch_with_jaccard gt = self._targets[img_id, cat_id] dt = self._predictions[img_id, cat_id] From 03430699f9b267a454badf03deee71a186d11e65 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 6 Jul 2025 02:07:52 +0000 Subject: [PATCH 20/24] Expanded all test cases in `test_coco_annotations_to_detections` including `use_iscrowd=True` and `use_iscrowd=False`. --- test/dataset/formats/test_coco.py | 226 +++++++++++++++++++++++++++++- 1 file changed, 225 insertions(+), 1 deletion(-) diff --git a/test/dataset/formats/test_coco.py b/test/dataset/formats/test_coco.py index 3a68894a..7c935a49 100644 --- a/test/dataset/formats/test_coco.py +++ b/test/dataset/formats/test_coco.py @@ -162,12 +162,21 @@ def test_group_coco_annotations_by_image_id( @pytest.mark.parametrize( - "image_annotations, resolution_wh, with_masks, expected_result, exception", + "image_annotations, resolution_wh, with_masks, use_iscrowd, expected_result, exception", [ ( [], (1000, 1000), False, + False, + Detections.empty(), + DoesNotRaise(), + ), # empty image annotations + ( + [], + (1000, 1000), + False, + True, Detections.empty(), DoesNotRaise(), ), # empty image annotations @@ -179,12 +188,32 @@ def test_group_coco_annotations_by_image_id( ], (1000, 1000), False, + False, Detections( xyxy=np.array([[0, 0, 100, 100]], dtype=np.float32), class_id=np.array([0], dtype=int), ), DoesNotRaise(), ), # single image annotations + ( + [ + mock_coco_annotation( + category_id=0, bbox=(0, 0, 100, 100), area=100 * 100 + ) + ], + (1000, 1000), + False, + True, + Detections( + xyxy=np.array([[0, 0, 100, 100]], dtype=np.float32), + class_id=np.array([0], dtype=int), + data={ + "iscrowd": np.array([0], dtype=int), + "area": np.array([100 * 100]), + }, + ), + DoesNotRaise(), + ), ( [ mock_coco_annotation( @@ -196,6 +225,7 @@ def test_group_coco_annotations_by_image_id( ], (1000, 1000), False, + False, Detections( xyxy=np.array( [[0, 0, 100, 100], [100, 100, 200, 200]], dtype=np.float32 @@ -204,6 +234,30 @@ def test_group_coco_annotations_by_image_id( ), DoesNotRaise(), ), # two image annotations + ( + [ + mock_coco_annotation( + category_id=0, bbox=(0, 0, 100, 100), area=100 * 100 + ), + mock_coco_annotation( + category_id=0, bbox=(100, 100, 100, 100), area=100 * 100 + ), + ], + (1000, 1000), + False, + True, + Detections( + xyxy=np.array( + [[0, 0, 100, 100], [100, 100, 200, 200]], dtype=np.float32 + ), + class_id=np.array([0, 0], dtype=int), + data={ + "iscrowd": np.array([0, 0], dtype=int), + "area": np.array([100 * 100, 100 * 100]), + }, + ), + DoesNotRaise(), + ), ( [ mock_coco_annotation( @@ -215,6 +269,7 @@ def test_group_coco_annotations_by_image_id( ], (5, 5), True, + False, Detections( xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32), class_id=np.array([0], dtype=int), @@ -232,6 +287,36 @@ def test_group_coco_annotations_by_image_id( ), DoesNotRaise(), ), # single image annotations with mask as polygon + ( + [ + mock_coco_annotation( + category_id=0, + bbox=(0, 0, 5, 5), + area=5 * 5, + segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]], + ) + ], + (5, 5), + True, + True, + Detections( + xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32), + class_id=np.array([0], dtype=int), + mask=np.array( + [ + [ + [1, 1, 1, 0, 0], + [1, 1, 1, 0, 0], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + ] + ] + ), + data={"iscrowd": np.array([0], dtype=int), "area": np.array([25])}, + ), + DoesNotRaise(), + ), ( [ mock_coco_annotation( @@ -247,6 +332,7 @@ def test_group_coco_annotations_by_image_id( ], (5, 5), True, + False, Detections( xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32), class_id=np.array([0], dtype=int), @@ -264,6 +350,40 @@ def test_group_coco_annotations_by_image_id( ), DoesNotRaise(), ), # single image annotations with mask, RLE segmentation mask + ( + [ + mock_coco_annotation( + category_id=0, + bbox=(0, 0, 5, 5), + area=5 * 5, + segmentation={ + "size": [5, 5], + "counts": [0, 15, 2, 3, 2, 3], + }, + iscrowd=True, + ) + ], + (5, 5), + True, + True, + Detections( + xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32), + class_id=np.array([0], dtype=int), + mask=np.array( + [ + [ + [1, 1, 1, 0, 0], + [1, 1, 1, 0, 0], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + ] + ] + ), + data={"iscrowd": np.array([1], dtype=int), "area": np.array([25])}, + ), + DoesNotRaise(), + ), ( [ mock_coco_annotation( @@ -285,6 +405,7 @@ def test_group_coco_annotations_by_image_id( ], (5, 5), True, + False, Detections( xyxy=np.array([[0, 0, 5, 5], [3, 0, 5, 2]], dtype=np.float32), class_id=np.array([0, 0], dtype=int), @@ -309,6 +430,56 @@ def test_group_coco_annotations_by_image_id( ), DoesNotRaise(), ), # two image annotations with mask, one mask as polygon and second as RLE + ( + [ + mock_coco_annotation( + category_id=0, + bbox=(0, 0, 5, 5), + area=5 * 5, + segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]], + ), + mock_coco_annotation( + category_id=0, + bbox=(3, 0, 2, 2), + area=2 * 2, + segmentation={ + "size": [5, 5], + "counts": [15, 2, 3, 2, 3], + }, + iscrowd=True, + ), + ], + (5, 5), + True, + True, + Detections( + xyxy=np.array([[0, 0, 5, 5], [3, 0, 5, 2]], dtype=np.float32), + class_id=np.array([0, 0], dtype=int), + mask=np.array( + [ + [ + [1, 1, 1, 0, 0], + [1, 1, 1, 0, 0], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + ], + [ + [0, 0, 0, 1, 1], + [0, 0, 0, 1, 1], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + ] + ), + data={ + "iscrowd": np.array([0, 1], dtype=int), + "area": np.array([25, 4]), + }, + ), + DoesNotRaise(), + ), # two image annotations with mask, one mask as polygon with iscrowd, and second as RLE without iscrowd ( [ mock_coco_annotation( @@ -330,6 +501,7 @@ def test_group_coco_annotations_by_image_id( ], (5, 5), True, + False, Detections( xyxy=np.array([[3, 0, 5, 2], [0, 0, 5, 5]], dtype=np.float32), class_id=np.array([0, 1], dtype=int), @@ -354,12 +526,63 @@ def test_group_coco_annotations_by_image_id( ), DoesNotRaise(), ), # two image annotations with mask, first mask as RLE and second as polygon + ( + [ + mock_coco_annotation( + category_id=0, + bbox=(3, 0, 2, 2), + area=2 * 2, + segmentation={ + "size": [5, 5], + "counts": [15, 2, 3, 2, 3], + }, + iscrowd=True, + ), + mock_coco_annotation( + category_id=1, + bbox=(0, 0, 5, 5), + area=5 * 5, + segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]], + ), + ], + (5, 5), + True, + True, + Detections( + xyxy=np.array([[3, 0, 5, 2], [0, 0, 5, 5]], dtype=np.float32), + class_id=np.array([0, 1], dtype=int), + mask=np.array( + [ + [ + [0, 0, 0, 1, 1], + [0, 0, 0, 1, 1], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0], + ], + [ + [1, 1, 1, 0, 0], + [1, 1, 1, 0, 0], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + [1, 1, 1, 1, 1], + ], + ] + ), + data={ + "iscrowd": np.array([1, 0], dtype=int), + "area": np.array([4, 25]), + }, + ), + DoesNotRaise(), + ), # two image annotations with mask, first mask as RLE with is crowd, and second as polygon without iscrowd ], ) def test_coco_annotations_to_detections( image_annotations: List[dict], resolution_wh: Tuple[int, int], with_masks: bool, + use_iscrowd: bool, expected_result: Detections, exception: Exception, ) -> None: @@ -368,6 +591,7 @@ def test_coco_annotations_to_detections( image_annotations=image_annotations, resolution_wh=resolution_wh, with_masks=with_masks, + use_iscrowd=use_iscrowd, ) assert result == expected_result From a95a81844133ffe28c1aecb19ab14f510fb02820 Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 6 Jul 2025 03:02:07 +0000 Subject: [PATCH 21/24] Including function `get_coco_class_index_mapping` in `supervision.__init__.py`. Unifying default value for parameter `use_iscrowd`. Adding detailed docstrings to function `get_coco_class_index_mapping`. --- supervision/__init__.py | 2 ++ supervision/dataset/formats/coco.py | 32 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/supervision/__init__.py b/supervision/__init__.py index 768212a5..a377c081 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -41,6 +41,7 @@ from supervision.dataset.core import ( ClassificationDataset, DetectionDataset, ) +from supervision.dataset.formats.coco import get_coco_class_index_mapping from supervision.dataset.utils import mask_to_rle, rle_to_mask from supervision.detection.core import Detections from supervision.detection.line_zone import ( @@ -204,6 +205,7 @@ __all__ = [ "draw_rectangle", "draw_text", "filter_polygons_by_area", + "get_coco_class_index_mapping", "get_polygon_center", "get_video_frames_generator", "letterbox_image", diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py index a0da8693..879ecfb2 100644 --- a/supervision/dataset/formats/coco.py +++ b/supervision/dataset/formats/coco.py @@ -93,7 +93,7 @@ def coco_annotations_to_detections( image_annotations: List[dict], resolution_wh: Tuple[int, int], with_masks: bool, - use_iscrowd: bool = False, + use_iscrowd: bool = True, ) -> Detections: if not image_annotations: return Detections.empty() @@ -174,6 +174,36 @@ def detections_to_coco_annotations( def get_coco_class_index_mapping(annotations_path: str) -> Dict[int, int]: + """ + Generates a mapping from sequential class indices to original COCO class ids. + + This function is essential when working with models that expect class ids to be + zero-indexed and sequential (0 to 79), as opposed to the original COCO + dataset where category ids are non-contiguous ranging from 1 to 90 but skipping some + ids. + + Use Cases: + - Evaluating models trained with COCO-style annotations where class ids + are sequential ranging from 0 to 79. + - Ensuring consistent class indexing across training, inference and evaluation, + when using different tools or datasets with COCO format. + - Reproducing results from models that assume sequential class ids (0 to 79). + + How it Works: + - Reads the COCO annotation file in its original format (`annotations_path`). + - Extracts and sorts all class names by their original COCO id (1 to 90). + - Builds a mapping from COCO class ids (not sequential with skipped ids) to + new class ids (sequential ranging from 0 to 79). + - Returns a dictionary mapping: `{new_class_id: original_COCO_class_id}`. + + Args: + annotations_path (str): Path to COCO JSON annotations file + (e.g., `instances_val2017.json`). + + Returns: + Dict[int, int]: A mapping from new class id (sequential ranging from 0 to 79) + to original COCO class id (1 to 90 with skipped ids). + """ coco_data = read_json_file(annotations_path) classes = coco_categories_to_classes(coco_categories=coco_data["categories"]) class_mapping = build_coco_class_index_mapping( From b867968a1083c5f92fd2a3ff329d8bb62b46df7a Mon Sep 17 00:00:00 2001 From: rafaelpadilla Date: Sun, 6 Jul 2025 03:33:30 +0000 Subject: [PATCH 22/24] breaking long lines, preventing E501. --- test/dataset/formats/test_coco.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/dataset/formats/test_coco.py b/test/dataset/formats/test_coco.py index 7c935a49..b8f53461 100644 --- a/test/dataset/formats/test_coco.py +++ b/test/dataset/formats/test_coco.py @@ -162,7 +162,8 @@ def test_group_coco_annotations_by_image_id( @pytest.mark.parametrize( - "image_annotations, resolution_wh, with_masks, use_iscrowd, expected_result, exception", + "image_annotations, resolution_wh, with_masks, use_iscrowd, " + "expected_result, exception", [ ( [], @@ -479,7 +480,8 @@ def test_group_coco_annotations_by_image_id( }, ), DoesNotRaise(), - ), # two image annotations with mask, one mask as polygon with iscrowd, and second as RLE without iscrowd + ), # two image annotations with mask, one mask as polygon with iscrowd, + # and second as RLE without iscrowd ( [ mock_coco_annotation( @@ -575,7 +577,8 @@ def test_group_coco_annotations_by_image_id( }, ), DoesNotRaise(), - ), # two image annotations with mask, first mask as RLE with is crowd, and second as polygon without iscrowd + ), # two image annotations with mask, first mask as RLE with is crowd, + # and second as polygon without iscrowd ], ) def test_coco_annotations_to_detections( From fcb47f7226591ae71ee853e27486046a7c66c984 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 7 Jul 2025 15:30:05 +0200 Subject: [PATCH 23/24] mkdocs update + pre-commit fix --- docs/detection/utils.md | 6 ++++++ docs/metrics/mean_average_precision.md | 6 ++++++ supervision/__init__.py | 3 --- supervision/utils/image.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/detection/utils.md b/docs/detection/utils.md index b0f1416f..323b2954 100644 --- a/docs/detection/utils.md +++ b/docs/detection/utils.md @@ -11,6 +11,12 @@ status: new :::supervision.detection.utils.box_iou_batch +
+

box_iou_batch_with_jaccard

+
+ +:::supervision.detection.utils.box_iou_batch_with_jaccard +

mask_iou_batch

diff --git a/docs/metrics/mean_average_precision.md b/docs/metrics/mean_average_precision.md index 817591a1..e2437c85 100644 --- a/docs/metrics/mean_average_precision.md +++ b/docs/metrics/mean_average_precision.md @@ -16,3 +16,9 @@ status: new :::supervision.metrics.mean_average_precision.MeanAveragePrecisionResult + +
+

get_coco_class_index_mapping

+
+ +:::supervision.dataset.formats.coco.get_coco_class_index_mapping \ No newline at end of file diff --git a/supervision/__init__.py b/supervision/__init__.py index a377c081..be7085df 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -1,7 +1,4 @@ import importlib.metadata as importlib_metadata -from typing import List - -import numpy as np try: # This will read version from pyproject.toml diff --git a/supervision/utils/image.py b/supervision/utils/image.py index 0bd2dbaa..37025656 100644 --- a/supervision/utils/image.py +++ b/supervision/utils/image.py @@ -785,4 +785,4 @@ def _merge_tiles_elements( def _generate_color_image( shape: Tuple[int, int], color: Tuple[int, int, int] ) -> np.ndarray: - return np.ones(shape[::-1] + (3,), dtype=np.uint8) * color + return np.ones((*shape[::-1], 3), dtype=np.uint8) * color From 5aa6b6f959cdd0bce39dd842cc23e58404a3e99e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:30:24 +0000 Subject: [PATCH 24/24] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/metrics/mean_average_precision.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/metrics/mean_average_precision.md b/docs/metrics/mean_average_precision.md index e2437c85..ce3e06a4 100644 --- a/docs/metrics/mean_average_precision.md +++ b/docs/metrics/mean_average_precision.md @@ -21,4 +21,4 @@ status: new

get_coco_class_index_mapping

-:::supervision.dataset.formats.coco.get_coco_class_index_mapping \ No newline at end of file +:::supervision.dataset.formats.coco.get_coco_class_index_mapping