diff --git a/docs/detection/utils.md b/docs/detection/utils.md
index b0f1416f..323b2954 100644
--- a/docs/detection/utils.md
+++ b/docs/detection/utils.md
@@ -11,6 +11,12 @@ status: new
:::supervision.detection.utils.box_iou_batch
+
+
+:::supervision.detection.utils.box_iou_batch_with_jaccard
+
diff --git a/docs/metrics/mean_average_precision.md b/docs/metrics/mean_average_precision.md
index 817591a1..ce3e06a4 100644
--- a/docs/metrics/mean_average_precision.md
+++ b/docs/metrics/mean_average_precision.md
@@ -16,3 +16,9 @@ status: new
:::supervision.metrics.mean_average_precision.MeanAveragePrecisionResult
+
+
+
+:::supervision.dataset.formats.coco.get_coco_class_index_mapping
diff --git a/supervision/__init__.py b/supervision/__init__.py
index 1a23702a..3a11f76a 100644
--- a/supervision/__init__.py
+++ b/supervision/__init__.py
@@ -38,6 +38,7 @@ from supervision.dataset.core import (
ClassificationDataset,
DetectionDataset,
)
+from supervision.dataset.formats.coco import get_coco_class_index_mapping
from supervision.dataset.utils import mask_to_rle, rle_to_mask
from supervision.detection.core import Detections
from supervision.detection.line_zone import (
@@ -58,6 +59,7 @@ from supervision.detection.tools.polygon_zone import PolygonZone, PolygonZoneAnn
from supervision.detection.tools.smoother import DetectionsSmoother
from supervision.detection.utils import (
box_iou_batch,
+ box_iou_batch_with_jaccard,
calculate_masks_centroids,
clip_boxes,
contains_holes,
@@ -180,6 +182,7 @@ __all__ = [
"VideoInfo",
"VideoSink",
"box_iou_batch",
+ "box_iou_batch_with_jaccard",
"box_non_max_merge",
"box_non_max_suppression",
"calculate_masks_centroids",
@@ -199,6 +202,7 @@ __all__ = [
"draw_rectangle",
"draw_text",
"filter_polygons_by_area",
+ "get_coco_class_index_mapping",
"get_polygon_center",
"get_video_frames_generator",
"letterbox_image",
diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py
index 8af54879..c6fc760e 100644
--- a/supervision/dataset/core.py
+++ b/supervision/dataset/core.py
@@ -574,7 +574,6 @@ class DetectionDataset(BaseDataset):
force_masks (bool): If True,
forces masks to be loaded for all annotations,
regardless of whether they are present.
-
Returns:
DetectionDataset: A DetectionDataset instance containing
the loaded images and annotations.
diff --git a/supervision/dataset/formats/coco.py b/supervision/dataset/formats/coco.py
index 9953d748..879ecfb2 100644
--- a/supervision/dataset/formats/coco.py
+++ b/supervision/dataset/formats/coco.py
@@ -90,7 +90,10 @@ def coco_annotations_to_masks(
def coco_annotations_to_detections(
- image_annotations: List[dict], resolution_wh: Tuple[int, int], with_masks: bool
+ image_annotations: List[dict],
+ resolution_wh: Tuple[int, int],
+ with_masks: bool,
+ use_iscrowd: bool = True,
) -> Detections:
if not image_annotations:
return Detections.empty()
@@ -102,15 +105,26 @@ def coco_annotations_to_detections(
xyxy = np.asarray(xyxy)
xyxy[:, 2:4] += xyxy[:, 0:2]
+ data = dict()
+ if use_iscrowd:
+ iscrowd = [
+ image_annotation["iscrowd"] for image_annotation in image_annotations
+ ]
+ area = [image_annotation["area"] for image_annotation in image_annotations]
+ data = dict(
+ iscrowd=np.asarray(iscrowd, dtype=int), area=np.asarray(area, dtype=float)
+ )
+
if with_masks:
mask = coco_annotations_to_masks(
image_annotations=image_annotations, resolution_wh=resolution_wh
)
- return Detections(
- class_id=np.asarray(class_ids, dtype=int), xyxy=xyxy, mask=mask
- )
+ else:
+ mask = None
- return Detections(xyxy=xyxy, class_id=np.asarray(class_ids, dtype=int))
+ return Detections(
+ class_id=np.asarray(class_ids, dtype=int), xyxy=xyxy, mask=mask, data=data
+ )
def detections_to_coco_annotations(
@@ -159,16 +173,58 @@ def detections_to_coco_annotations(
return coco_annotations, annotation_id
+def get_coco_class_index_mapping(annotations_path: str) -> Dict[int, int]:
+ """
+ Generates a mapping from sequential class indices to original COCO class ids.
+
+ This function is essential when working with models that expect class ids to be
+ zero-indexed and sequential (0 to 79), as opposed to the original COCO
+ dataset where category ids are non-contiguous ranging from 1 to 90 but skipping some
+ ids.
+
+ Use Cases:
+ - Evaluating models trained with COCO-style annotations where class ids
+ are sequential ranging from 0 to 79.
+ - Ensuring consistent class indexing across training, inference and evaluation,
+ when using different tools or datasets with COCO format.
+ - Reproducing results from models that assume sequential class ids (0 to 79).
+
+ How it Works:
+ - Reads the COCO annotation file in its original format (`annotations_path`).
+ - Extracts and sorts all class names by their original COCO id (1 to 90).
+ - Builds a mapping from COCO class ids (not sequential with skipped ids) to
+ new class ids (sequential ranging from 0 to 79).
+ - Returns a dictionary mapping: `{new_class_id: original_COCO_class_id}`.
+
+ Args:
+ annotations_path (str): Path to COCO JSON annotations file
+ (e.g., `instances_val2017.json`).
+
+ Returns:
+ Dict[int, int]: A mapping from new class id (sequential ranging from 0 to 79)
+ to original COCO class id (1 to 90 with skipped ids).
+ """
+ coco_data = read_json_file(annotations_path)
+ classes = coco_categories_to_classes(coco_categories=coco_data["categories"])
+ class_mapping = build_coco_class_index_mapping(
+ coco_categories=coco_data["categories"], target_classes=classes
+ )
+ return {v: k for k, v in class_mapping.items()}
+
+
def load_coco_annotations(
images_directory_path: str,
annotations_path: str,
force_masks: bool = False,
+ use_iscrowd: bool = True,
) -> Tuple[List[str], List[str], Dict[str, Detections]]:
coco_data = read_json_file(file_path=annotations_path)
classes = coco_categories_to_classes(coco_categories=coco_data["categories"])
+
class_index_mapping = build_coco_class_index_mapping(
coco_categories=coco_data["categories"], target_classes=classes
)
+
coco_images = coco_data["images"]
coco_annotations_groups = group_coco_annotations_by_image_id(
coco_annotations=coco_data["annotations"]
@@ -190,7 +246,9 @@ def load_coco_annotations(
image_annotations=image_annotations,
resolution_wh=(image_width, image_height),
with_masks=force_masks,
+ use_iscrowd=use_iscrowd,
)
+
annotation = map_detections_class_id(
source_to_target_mapping=class_index_mapping,
detections=annotation,
diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py
index 8153628b..d86990ee 100644
--- a/supervision/detection/utils.py
+++ b/supervision/detection/utils.py
@@ -1325,3 +1325,103 @@ def spread_out_boxes(
xyxy_padded[:, [2, 3]] += force_vectors
return pad_boxes(xyxy_padded, px=-1)
+
+
+def _jaccard(box_a: List[float], box_b: List[float], is_crowd: bool) -> float:
+ """
+ Calculate the Jaccard index (intersection over union) between two bounding boxes.
+ If a gt object is marked as "iscrowd", a dt is allowed to match any subregion
+ of the gt. Choosing gt' in the crowd gt that best matches the dt can be done using
+ gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing
+ iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt)
+
+ Args:
+ box_a (List[float]): Box coordinates in the format [x, y, width, height].
+ box_b (List[float]): Box coordinates in the format [x, y, width, height].
+ iscrowd (bool): Flag indicating if the second box is a crowd region or not.
+
+ Returns:
+ float: Jaccard index between the two bounding boxes.
+ """
+ # Smallest number to avoid division by zero
+ EPS = np.spacing(1)
+
+ xa, ya, x2a, y2a = box_a[0], box_a[1], box_a[0] + box_a[2], box_a[1] + box_a[3]
+ xb, yb, x2b, y2b = box_b[0], box_b[1], box_b[0] + box_b[2], box_b[1] + box_b[3]
+
+ # Innermost left x
+ xi = max(xa, xb)
+ # Innermost right x
+ x2i = min(x2a, x2b)
+ # Same for y
+ yi = max(ya, yb)
+ y2i = min(y2a, y2b)
+
+ # Calculate areas
+ Aa = max(x2a - xa, 0.0) * max(y2a - ya, 0.0)
+ Ab = max(x2b - xb, 0.0) * max(y2b - yb, 0.0)
+ Ai = max(x2i - xi, 0.0) * max(y2i - yi, 0.0)
+
+ if is_crowd:
+ return Ai / (Aa + EPS)
+
+ return Ai / (Aa + Ab - Ai + EPS)
+
+
+def box_iou_batch_with_jaccard(
+ boxes_true: List[List[float]],
+ boxes_detection: List[List[float]],
+ is_crowd: List[bool],
+) -> np.ndarray:
+ """
+ Calculate the intersection over union (IoU) between detection bounding boxes (dt)
+ and ground-truth bounding boxes (gt).
+ Reference: https://github.com/rafaelpadilla/review_object_detection_metrics
+
+ Args:
+ boxes_true (List[List[float]]): List of ground-truth bounding boxes in the \
+ format [x, y, width, height].
+ boxes_detection (List[List[float]]): List of detection bounding boxes in the \
+ format [x, y, width, height].
+ is_crowd (List[bool]): List indicating if each ground-truth bounding box \
+ is a crowd region or not.
+
+ Returns:
+ np.ndarray: Array of IoU values of shape (len(dt), len(gt)).
+
+ Examples:
+ ```python
+ import numpy as np
+ import supervision as sv
+
+ boxes_true = [
+ [10, 20, 30, 40], # x, y, w, h
+ [15, 25, 35, 45]
+ ]
+ boxes_detection = [
+ [12, 22, 28, 38],
+ [16, 26, 36, 46]
+ ]
+ is_crowd = [False, False]
+
+ ious = sv.box_iou_batch_with_jaccard(
+ boxes_true=boxes_true,
+ boxes_detection=boxes_detection,
+ is_crowd=is_crowd
+ )
+ # array([
+ # [0.8866..., 0.4960...],
+ # [0.4000..., 0.8622...]
+ # ])
+ ```
+ """
+ assert len(is_crowd) == len(boxes_true), (
+ "`is_crowd` must have the same length as `boxes_true`"
+ )
+ if len(boxes_detection) == 0 or len(boxes_true) == 0:
+ return np.array([])
+ ious = np.zeros((len(boxes_detection), len(boxes_true)), dtype=np.float64)
+ for g_idx, g in enumerate(boxes_true):
+ for d_idx, d in enumerate(boxes_detection):
+ ious[d_idx, g_idx] = _jaccard(d, g, is_crowd[g_idx])
+ return ious
diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py
index 9e7a30d0..8e8d18d7 100644
--- a/supervision/metrics/mean_average_precision.py
+++ b/supervision/metrics/mean_average_precision.py
@@ -1,422 +1,27 @@
from __future__ import annotations
+import copy
+import datetime
+import itertools
+from collections import defaultdict
from copy import deepcopy
from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import numpy as np
from matplotlib import pyplot as plt
-from supervision.config import ORIENTED_BOX_COORDINATES
+from supervision import box_iou_batch_with_jaccard
from supervision.detection.core import Detections
-from supervision.detection.utils import (
- box_iou_batch,
- mask_iou_batch,
- oriented_box_iou_batch,
-)
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import Metric, MetricTarget
-from supervision.metrics.utils.object_size import (
- ObjectSizeCategory,
- get_detection_size_category,
-)
from supervision.metrics.utils.utils import ensure_pandas_installed
if TYPE_CHECKING:
import pandas as pd
-class MeanAveragePrecision(Metric):
- """
- Mean Average Precision (mAP) is a metric used to evaluate object detection models.
- It is the average of the precision-recall curves at different IoU thresholds.
-
- Example:
- ```python
- import supervision as sv
- from supervision.metrics import MeanAveragePrecision
-
- predictions = sv.Detections(...)
- targets = sv.Detections(...)
-
- map_metric = MeanAveragePrecision()
- map_result = map_metric.update(predictions, targets).compute()
-
- print(map_result.map50_95)
- # 0.4674
-
- print(map_result)
- # MeanAveragePrecisionResult:
- # Metric target: MetricTarget.BOXES
- # Class agnostic: False
- # mAP @ 50:95: 0.4674
- # mAP @ 50: 0.5048
- # mAP @ 75: 0.4796
- # mAP scores: [0.50485 0.50377 0.50377 ...]
- # IoU thresh: [0.5 0.55 0.6 ...]
- # AP per class:
- # 0: [0.67699 0.67699 0.67699 ...]
- # ...
- # Small objects: ...
- # Medium objects: ...
- # Large objects: ...
-
- map_result.plot()
- ```
-
- { align=center width="800" }
- """
-
- def __init__(
- self,
- metric_target: MetricTarget = MetricTarget.BOXES,
- class_agnostic: bool = False,
- ):
- """
- Initialize the Mean Average Precision metric.
-
- Args:
- metric_target (MetricTarget): The type of detection data to use.
- class_agnostic (bool): Whether to treat all data as a single class.
- """
- self._metric_target = metric_target
- self._class_agnostic = class_agnostic
-
- self._predictions_list: List[Detections] = []
- self._targets_list: List[Detections] = []
-
- def reset(self) -> None:
- """
- Reset the metric to its initial state, clearing all stored data.
- """
- self._predictions_list = []
- self._targets_list = []
-
- def update(
- self,
- predictions: Union[Detections, List[Detections]],
- targets: Union[Detections, List[Detections]],
- ) -> MeanAveragePrecision:
- """
- Add new predictions and targets to the metric, but do not compute the result.
-
- Args:
- predictions (Union[Detections, List[Detections]]): The predicted detections.
- targets (Union[Detections, List[Detections]]): The ground-truth detections.
-
- Returns:
- (MeanAveragePrecision): The updated metric instance.
- """
- if not isinstance(predictions, list):
- predictions = [predictions]
- if not isinstance(targets, list):
- targets = [targets]
-
- if len(predictions) != len(targets):
- raise ValueError(
- f"The number of predictions ({len(predictions)}) and"
- f" targets ({len(targets)}) during the update must be the same."
- )
-
- if self._class_agnostic:
- predictions = deepcopy(predictions)
- targets = deepcopy(targets)
-
- for prediction in predictions:
- prediction.class_id[:] = -1
- for target in targets:
- target.class_id[:] = -1
-
- self._predictions_list.extend(predictions)
- self._targets_list.extend(targets)
-
- return self
-
- def compute(
- self,
- ) -> MeanAveragePrecisionResult:
- """
- Calculate Mean Average Precision based on predicted and ground-truth
- detections at different thresholds.
-
- Returns:
- (MeanAveragePrecisionResult): The Mean Average Precision result.
- """
- result = self._compute(self._predictions_list, self._targets_list)
-
- small_predictions = []
- small_targets = []
- for predictions, targets in zip(self._predictions_list, self._targets_list):
- small_predictions.append(
- self._filter_detections_by_size(predictions, ObjectSizeCategory.SMALL)
- )
- small_targets.append(
- self._filter_detections_by_size(targets, ObjectSizeCategory.SMALL)
- )
- result.small_objects = self._compute(small_predictions, small_targets)
-
- medium_predictions = []
- medium_targets = []
- for predictions, targets in zip(self._predictions_list, self._targets_list):
- medium_predictions.append(
- self._filter_detections_by_size(predictions, ObjectSizeCategory.MEDIUM)
- )
- medium_targets.append(
- self._filter_detections_by_size(targets, ObjectSizeCategory.MEDIUM)
- )
- result.medium_objects = self._compute(medium_predictions, medium_targets)
-
- large_predictions = []
- large_targets = []
- for predictions, targets in zip(self._predictions_list, self._targets_list):
- large_predictions.append(
- self._filter_detections_by_size(predictions, ObjectSizeCategory.LARGE)
- )
- large_targets.append(
- self._filter_detections_by_size(targets, ObjectSizeCategory.LARGE)
- )
- result.large_objects = self._compute(large_predictions, large_targets)
-
- return result
-
- def _compute(
- self,
- predictions_list: List[Detections],
- targets_list: List[Detections],
- ) -> MeanAveragePrecisionResult:
- iou_thresholds = np.linspace(0.5, 0.95, 10)
- stats = []
-
- for predictions, targets in zip(predictions_list, targets_list):
- prediction_contents = self._detections_content(predictions)
- target_contents = self._detections_content(targets)
-
- if len(targets) > 0:
- if len(predictions) == 0:
- stats.append(
- (
- np.zeros((0, iou_thresholds.size), dtype=bool),
- np.zeros((0,), dtype=np.float32),
- np.zeros((0,), dtype=int),
- targets.class_id,
- )
- )
-
- else:
- if self._metric_target == MetricTarget.BOXES:
- iou = box_iou_batch(target_contents, prediction_contents)
- elif self._metric_target == MetricTarget.MASKS:
- iou = mask_iou_batch(target_contents, prediction_contents)
- elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- iou = oriented_box_iou_batch(
- target_contents, prediction_contents
- )
- else:
- raise ValueError(
- "Unsupported metric target for IoU calculation"
- )
-
- matches = self._match_detection_batch(
- predictions.class_id, targets.class_id, iou, iou_thresholds
- )
-
- stats.append(
- (
- matches,
- predictions.confidence,
- predictions.class_id,
- targets.class_id,
- )
- )
-
- # Compute average precisions if any matches exist
- if stats:
- concatenated_stats = [np.concatenate(items, 0) for items in zip(*stats)]
- average_precisions, unique_classes = self._average_precisions_per_class(
- *concatenated_stats
- )
- mAP_scores = np.mean(average_precisions, axis=0)
- else:
- mAP_scores = np.zeros((10,), dtype=np.float32)
- unique_classes = np.empty((0,), dtype=int)
- average_precisions = np.empty((0, len(iou_thresholds)), dtype=np.float32)
-
- return MeanAveragePrecisionResult(
- metric_target=self._metric_target,
- is_class_agnostic=self._class_agnostic,
- mAP_scores=mAP_scores,
- iou_thresholds=iou_thresholds,
- matched_classes=unique_classes,
- ap_per_class=average_precisions,
- )
-
- @staticmethod
- def _compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float:
- """
- Compute the average precision using 101-point interpolation (COCO), given
- the recall and precision curves.
-
- Args:
- recall (np.ndarray): The recall curve.
- precision (np.ndarray): The precision curve.
-
- Returns:
- (float): Average precision.
- """
- if len(recall) == 0 and len(precision) == 0:
- return 0.0
-
- recall_levels = np.linspace(0, 1, 101)
- precision_levels = np.zeros_like(recall_levels)
- for r, p in zip(recall[::-1], precision[::-1]):
- precision_levels[recall_levels <= r] = p
-
- average_precision = (1 / 101 * precision_levels).sum()
- return average_precision
-
- @staticmethod
- def _match_detection_batch(
- predictions_classes: np.ndarray,
- target_classes: np.ndarray,
- iou: np.ndarray,
- iou_thresholds: np.ndarray,
- ) -> np.ndarray:
- num_predictions, num_iou_levels = (
- predictions_classes.shape[0],
- iou_thresholds.shape[0],
- )
- correct = np.zeros((num_predictions, num_iou_levels), dtype=bool)
- correct_class = target_classes[:, None] == predictions_classes
-
- for i, iou_level in enumerate(iou_thresholds):
- matched_indices = np.where((iou >= iou_level) & correct_class)
-
- if matched_indices[0].shape[0]:
- combined_indices = np.stack(matched_indices, axis=1)
- iou_values = iou[matched_indices][:, None]
- matches = np.hstack([combined_indices, iou_values])
-
- if matched_indices[0].shape[0] > 1:
- matches = matches[matches[:, 2].argsort()[::-1]]
- matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
- matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
-
- correct[matches[:, 1].astype(int), i] = True
-
- return correct
-
- @staticmethod
- def _average_precisions_per_class(
- matches: np.ndarray,
- prediction_confidence: np.ndarray,
- prediction_class_ids: np.ndarray,
- true_class_ids: np.ndarray,
- ) -> Tuple[np.ndarray, np.ndarray]:
- """
- Compute the average precision, given the recall and precision curves.
- Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
-
- Args:
- matches (np.ndarray): True positives.
- prediction_confidence (np.ndarray): Objectness value from 0-1.
- prediction_class_ids (np.ndarray): Predicted object classes.
- true_class_ids (np.ndarray): True object classes.
- eps (float, optional): Small value to prevent division by zero.
-
- Returns:
- (Tuple[np.ndarray, np.ndarray]): Average precision for different
- IoU levels, and an array of class IDs that were matched.
- """
- eps = 1e-16
-
- sorted_indices = np.argsort(-prediction_confidence)
- matches = matches[sorted_indices]
- prediction_class_ids = prediction_class_ids[sorted_indices]
-
- unique_classes, class_counts = np.unique(true_class_ids, return_counts=True)
- num_classes = unique_classes.shape[0]
-
- average_precisions = np.zeros((num_classes, matches.shape[1]))
-
- for class_idx, class_id in enumerate(unique_classes):
- is_class = prediction_class_ids == class_id
- total_true = class_counts[class_idx]
- total_prediction = is_class.sum()
-
- if total_prediction == 0 or total_true == 0:
- continue
-
- false_positives = (1 - matches[is_class]).cumsum(0)
- true_positives = matches[is_class].cumsum(0)
- false_negatives = total_true - true_positives
-
- recall = true_positives / (true_positives + false_negatives + eps)
- precision = true_positives / (true_positives + false_positives)
-
- for iou_level_idx in range(matches.shape[1]):
- average_precisions[class_idx, iou_level_idx] = (
- MeanAveragePrecision._compute_average_precision(
- recall[:, iou_level_idx], precision[:, iou_level_idx]
- )
- )
-
- return average_precisions, unique_classes
-
- def _detections_content(self, detections: Detections) -> np.ndarray:
- """Return boxes, masks or oriented bounding boxes from detections."""
- if self._metric_target == MetricTarget.BOXES:
- return detections.xyxy
- if self._metric_target == MetricTarget.MASKS:
- return (
- detections.mask
- if detections.mask is not None
- else self._make_empty_content()
- )
- if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- obb = detections.data.get(ORIENTED_BOX_COORDINATES)
- if obb is not None and len(obb) > 0:
- return np.array(obb, dtype=np.float32)
- return self._make_empty_content()
- raise ValueError(f"Invalid metric target: {self._metric_target}")
-
- def _make_empty_content(self) -> np.ndarray:
- if self._metric_target == MetricTarget.BOXES:
- return np.empty((0, 4), dtype=np.float32)
- if self._metric_target == MetricTarget.MASKS:
- return np.empty((0, 0, 0), dtype=bool)
- if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
- return np.empty((0, 4, 2), dtype=np.float32)
- raise ValueError(f"Invalid metric target: {self._metric_target}")
-
- def _filter_detections_by_size(
- self, detections: Detections, size_category: ObjectSizeCategory
- ) -> Detections:
- """Return a copy of detections with contents filtered by object size."""
- new_detections = deepcopy(detections)
- if detections.is_empty() or size_category == ObjectSizeCategory.ANY:
- return new_detections
-
- sizes = get_detection_size_category(new_detections, self._metric_target)
- size_mask = sizes == size_category.value
-
- new_detections.xyxy = new_detections.xyxy[size_mask]
- if new_detections.mask is not None:
- new_detections.mask = new_detections.mask[size_mask]
- if new_detections.class_id is not None:
- new_detections.class_id = new_detections.class_id[size_mask]
- if new_detections.confidence is not None:
- new_detections.confidence = new_detections.confidence[size_mask]
- if new_detections.tracker_id is not None:
- new_detections.tracker_id = new_detections.tracker_id[size_mask]
- if new_detections.data is not None:
- for key, value in new_detections.data.items():
- new_detections.data[key] = np.array(value)[size_mask]
-
- return new_detections
-
-
@dataclass
class MeanAveragePrecisionResult:
"""
@@ -472,56 +77,34 @@ class MeanAveragePrecisionResult:
def __str__(self) -> str:
"""
- Format as a pretty string.
+ Formats the evaluation output metrics to match the structure used by pycocotools
Example:
- ```python
- print(map_result)
- # MeanAveragePrecisionResult:
- # Metric target: MetricTarget.BOXES
- # Class agnostic: False
- # mAP @ 50:95: 0.4674
- # mAP @ 50: 0.5048
- # mAP @ 75: 0.4796
- # mAP scores: [0.50485 0.50377 0.50377 ...]
- # IoU thresh: [0.5 0.55 0.6 ...]
- # AP per class:
- # 0: [0.67699 0.67699 0.67699 ...]
- # ...
- # Small objects: ...
- # Medium objects: ...
- # Large objects: ...
+ ```python
+ print(map_result)
+ # MeanAveragePrecisionResult:
+ Average Precision (AP) @[ IoU=0.50:0.95 | area= all | maxDets=100 ] = 0.464
+ Average Precision (AP) @[ IoU=0.50 | area= all | maxDets=100 ] = 0.637
+ Average Precision (AP) @[ IoU=0.75 | area= all | maxDets=100 ] = 0.203
+ Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.284
+ Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.497
+ Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.629
```
"""
-
- out_str = (
- f"{self.__class__.__name__}:\n"
- f"Metric target: {self.metric_target}\n"
- f"Class agnostic: {self.is_class_agnostic}\n"
- f"mAP @ 50:95: {self.map50_95:.4f}\n"
- f"mAP @ 50: {self.map50:.4f}\n"
- f"mAP @ 75: {self.map75:.4f}\n"
- f"mAP scores: {self.mAP_scores}\n"
- f"IoU thresh: {self.iou_thresholds}\n"
- f"AP per class:\n"
+ return (
+ f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | "
+ f"maxDets=100 ] = {self.map50_95:.3f}\n"
+ f"Average Precision (AP) @[ IoU=0.50 | area= all | "
+ f"maxDets=100 ] = {self.map50:.3f}\n"
+ f"Average Precision (AP) @[ IoU=0.75 | area= all | "
+ f"maxDets=100 ] = {self.map75:.3f}\n"
+ f"Average Precision (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] "
+ f"= {self.small_objects.map50_95:.3f}\n"
+ f"Average Precision (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] \
+ = {self.medium_objects.map50_95:.3f}\n"
+ f"Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] \
+ = {self.large_objects.map50_95:.3f}"
)
- if self.ap_per_class.size == 0:
- out_str += " No results\n"
- for class_id, ap_of_class in zip(self.matched_classes, self.ap_per_class):
- out_str += f" {class_id}: {ap_of_class}\n"
-
- indent = " "
- if self.small_objects is not None:
- indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
- out_str += f"\nSmall objects:\n{indented}"
- if self.medium_objects is not None:
- indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
- out_str += f"\nMedium objects:\n{indented}"
- if self.large_objects is not None:
- indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
- out_str += f"\nLarge objects:\n{indented}"
-
- return out_str
def to_pandas(self) -> "pd.DataFrame":
"""
@@ -553,7 +136,6 @@ class MeanAveragePrecisionResult:
pandas_data[f"large_objects_{key}"] = value
# Average precisions are currently not included in the DataFrame.
-
return pd.DataFrame(
pandas_data,
index=[0],
@@ -626,3 +208,1143 @@ class MeanAveragePrecisionResult:
plt.tight_layout()
plt.show()
+
+
+class EvaluationDataset:
+ """
+ Class used representing a dataset in the right format needed by the
+ `COCOEvaluator` class.
+
+ Reference: https://github.com/rafaelpadilla/review_object_detection_metrics
+ """
+
+ def __init__(self, targets: Optional[Dict[str, Any]] = None):
+ """
+ Constructor of EvaluationDataset object used to evaluate models with
+ Mean Average Precision.
+ Args:
+ targets (dict): The targets (ground truth) of the dataset in a the
+ COCO format.
+ """
+ # Initialize members
+ self.dataset, self.anns, self.cats, self.imgs = dict(), dict(), dict(), dict()
+ self.img_to_anns, self.cat_to_imgs = defaultdict(list), defaultdict(list)
+
+ if targets is None:
+ return
+
+ # Load dataset
+ self.dataset = targets
+ self.create_class_members()
+
+ @classmethod
+ def empty(cls):
+ return cls(targets=None)
+
+ def create_class_members(self):
+ """
+ Create index elements for the dataset.
+ """
+ anns, cats, imgs = {}, {}, {}
+ img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list)
+ if "annotations" in self.dataset:
+ for ann in self.dataset["annotations"]:
+ img_to_anns[ann["image_id"]].append(ann)
+ anns[ann["id"]] = ann
+
+ if "images" in self.dataset:
+ for img in self.dataset["images"]:
+ imgs[img["id"]] = img
+
+ if "categories" in self.dataset:
+ for cat in self.dataset["categories"]:
+ cats[cat["id"]] = cat
+
+ if "annotations" in self.dataset and "categories" in self.dataset:
+ for ann in self.dataset["annotations"]:
+ cat_to_imgs[ann["category_id"]].append(ann["image_id"])
+
+ # Populate class members
+ self.anns = anns
+ self.img_to_anns = img_to_anns
+ self.cat_to_imgs = cat_to_imgs
+ self.imgs = imgs
+ self.cats = cats
+
+ def get_annotation_ids(
+ self,
+ img_ids: List[int] = [],
+ cat_ids: List[int] = [],
+ area_range: Tuple[float, float] = [],
+ iscrowd: bool = False,
+ ):
+ """
+ Get annotation ids that satisfy given filter conditions.
+ Args:
+ img_ids (list): ids of the images that we want to retrieve.
+ cat_ids (list): ids of the categories that we want to retrieve.
+ area_range (tuple): area range of the annotations that we want to retrieve
+ in the format [min_area, max_area].
+ iscrowd (bool): if annotations to retrieve are `iscrowded=1`.
+ """
+ # If there are no filters, we use all annotations
+ if len(img_ids) == len(cat_ids) == len(area_range) == 0:
+ anns = self.dataset["annotations"]
+ else:
+ if len(img_ids) != 0:
+ lists = [
+ self.img_to_anns[img_id]
+ for img_id in img_ids
+ if img_id in self.img_to_anns
+ ]
+ anns = list(itertools.chain.from_iterable(lists))
+ else:
+ anns = self.dataset["annotations"]
+
+ # Filter by category
+ anns = (
+ anns
+ if len(cat_ids) == 0
+ else [ann for ann in anns if ann["category_id"] in cat_ids]
+ )
+
+ # Filter by area
+ anns = (
+ anns
+ if len(area_range) == 0
+ else [
+ ann
+ for ann in anns
+ if ann["area"] > area_range[0] and ann["area"] < area_range[1]
+ ]
+ )
+
+ # Filter by iscrowd
+ if iscrowd is True:
+ ids = [ann["id"] for ann in anns if ann["iscrowd"] == 1]
+ else:
+ ids = [ann["id"] for ann in anns]
+ return ids
+
+ def get_category_ids(
+ self,
+ cat_names: List[str] = [],
+ supercategory_names: List[str] = [],
+ cat_ids: List[int] = [],
+ ) -> List[int]:
+ """
+ Get category ids that satisfy given filter conditions.
+ Args:
+ cat_names (list): names of the categories to retrieve.
+ supercategory_names (list): names of the supercategories to retrieve.
+ cat_ids (list): ids of the categories to retrieve.
+ Returns:
+ ids (list): integer array of category ids.
+ """
+ # If there are no filters, we use all categories
+ if len(cat_names) == len(supercategory_names) == len(cat_ids) == 0:
+ cats = self.dataset["categories"]
+ else:
+ cats = self.dataset["categories"]
+
+ # Filter by name
+ cats = (
+ cats
+ if len(cat_names) == 0
+ else [cat for cat in cats if cat["name"] in cat_names]
+ )
+
+ # Filter by supercategory
+ cats = (
+ cats
+ if len(supercategory_names) == 0
+ else [
+ cat for cat in cats if cat["supercategory"] in supercategory_names
+ ]
+ )
+
+ # Filter by id
+ cats = (
+ cats
+ if len(cat_ids) == 0
+ else [cat for cat in cats if cat["id"] in cat_ids]
+ )
+ ids = [cat["id"] for cat in cats]
+ return ids
+
+ def get_image_ids(
+ self,
+ img_ids: List[int] = [],
+ cat_ids: List[int] = [],
+ ) -> List[int]:
+ """
+ Get image ids that satisfy given filter conditions.
+ Args:
+ img_ids (list): ids of the images to retrieve.
+ cat_ids (list): ids of the categories to retrieve.
+ Returns:
+ ids (list): integer array of image ids.
+ """
+ # If there are no filters, we use all images
+ if len(img_ids) == len(cat_ids) == 0:
+ ids = self.imgs.keys()
+ return list(ids)
+
+ ids = set(img_ids)
+ for i, cat_id in enumerate(cat_ids):
+ if i == 0 and len(ids) == 0:
+ ids = set(self.cat_to_imgs[cat_id])
+ else:
+ ids &= set(self.cat_to_imgs[cat_id])
+ return list(ids)
+
+ def get_annotations(self, ids: List[int] = []) -> List[dict]:
+ """
+ Get annotations with the specified ids.
+ Args:
+ ids (list): integer ids specifying annotations.
+ Returns:
+ anns (list): loaded annotations.
+ """
+ return [self.anns[idx] for idx in ids]
+
+ def load_predictions(self, predictions: List[Dict]) -> "EvaluationDataset":
+ """
+ Load prediction result into an EvaluationDataset object.
+ Args:
+ predictions (list): prediction result.
+ Returns:
+ EvaluationDataset object representing the predictions.
+ """
+ # Create an empty EvaluationDataset object for the predictions
+ predictions_dataset = EvaluationDataset.empty()
+ predictions_dataset.dataset["images"] = [img for img in self.dataset["images"]]
+
+ if not isinstance(predictions, list):
+ raise ValueError("results must be a list")
+
+ ids = [pred["image_id"] for pred in predictions]
+
+ # Make sure the image ids from predictions exist in the current dataset
+ assert set(ids) == (set(ids) & set(self.get_image_ids())), (
+ "Results do not correspond to current coco set"
+ )
+
+ # Check if the predictions contain any unsupported keys
+ if "caption" in predictions[0]:
+ raise NotImplementedError(
+ "Evaluating predictions with caption is not supported."
+ )
+ elif "segmentation" in predictions[0]:
+ raise NotImplementedError(
+ "Evaluating predictions with segmentation is not supported."
+ )
+ elif "keypoints" in predictions[0]:
+ raise NotImplementedError(
+ "Evaluating predictions with keypoints is not supported."
+ )
+
+ elif "bbox" in predictions[0] and not predictions[0]["bbox"] == []:
+ predictions_dataset.dataset["categories"] = copy.deepcopy(
+ self.dataset["categories"]
+ )
+
+ # Prepare fields for every prediction of the given image
+ for idx, pred in enumerate(predictions):
+ x, y, w, h = pred["bbox"]
+ x1, x2, y1, y2 = [x, x + w, y, y + h]
+
+ # Make segmentation from bounding box coordinates
+ if "segmentation" not in pred:
+ pred["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
+ pred["area"] = w * h
+ pred["id"] = idx + 1
+ # For predictions we set iscrowd to 0
+ pred["iscrowd"] = 0
+ predictions_dataset.dataset["annotations"] = predictions
+ predictions_dataset.create_class_members()
+ return predictions_dataset
+
+
+# Area ranges for object size in pixels
+SMALL_OBJECT_AREA = 32**2
+MEDIUM_OBJECT_AREA = 96**2
+MAX_ALL_OBJECT_AREA = 1e5**2
+
+# Smallest number to avoid division by zero
+EPS = np.spacing(1)
+
+
+class ObjectSize(Enum):
+ """
+ Enum for object size.
+ """
+
+ ALL = "all"
+ SMALL = "small"
+ MEDIUM = "medium"
+ LARGE = "large"
+
+
+class COCOEvaluatorParameters:
+ """
+ Parameters for COCOEvaluator
+ """
+
+ def __init__(self):
+ """Initialize all parameters for evaluation"""
+
+ self.img_ids, self.cat_ids = [], []
+ # IoU thresholds [0.5, 0.55, 0.6, 0.65, ..., 0.95]
+ self.iou_thrs = np.linspace(
+ 0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True
+ )
+ # 101 recall thresholds [0.0, 0.01, 0.02, ..., 1.00]
+ self.rec_thrs = np.linspace(
+ 0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True
+ )
+ # 3 maximum detection thresholds [1, 10, 100]
+ self.max_dets = [1, 10, 100]
+ # Area ranges [0, 1e5], [0, 32], [32, 96], [96, 1e5]
+ self.area_range = [
+ [0, MAX_ALL_OBJECT_AREA],
+ [0, SMALL_OBJECT_AREA],
+ [SMALL_OBJECT_AREA, MEDIUM_OBJECT_AREA],
+ [MEDIUM_OBJECT_AREA, MAX_ALL_OBJECT_AREA],
+ ]
+
+
+class COCOEvaluator:
+ """
+ Evaluator class to compute COCO metrics.
+ """
+
+ def __init__(
+ self, coco_targets: EvaluationDataset, coco_predictions: EvaluationDataset
+ ):
+ """
+ Constructor of COCOEvaluator object.
+
+ Args:
+ coco_targets (EvaluationDataset): The dataset with the ground truths.
+ coco_predictions (EvaluationDataset): The dataset with the predictions.
+ """
+ if coco_targets is None:
+ raise ValueError("coco_targets must be provided")
+ if coco_predictions is None:
+ raise ValueError("coco_predictions must be provided")
+
+ self.coco_targets = coco_targets
+ self.coco_predictions = coco_predictions
+ # List of dictionaries containing the evaluation results
+ # len(eval_imgs) = (categories) * (area_ranges) * (images)
+ # For COCO 2017: len(eval_images) = 80 * 4 * 5000 = 1600000
+ self.eval_imgs = defaultdict(list)
+ # Dictionary of accumulated results
+ self.results = {}
+ # Dictionary of targets for evaluation
+ self._targets = defaultdict(list)
+ self._predictions = defaultdict(list)
+ # Parameters for evaluation
+ self.params = COCOEvaluatorParameters()
+ # List of results summarization
+ self.stats = []
+ # Dictionary of IOUs between all targets and predictions
+ self.ious = {}
+ # Set image and category ids
+ self.params.img_ids = sorted(self.coco_targets.get_image_ids())
+ self.params.cat_ids = sorted(self.coco_targets.get_category_ids())
+
+ def _prepare_targets_and_predictions(self):
+ """
+ Prepare targets and predictions for evaluation.
+ """
+ # Get the target samples for the evaluation
+ annotation_ids = self.coco_targets.get_annotation_ids(
+ img_ids=self.params.img_ids, cat_ids=self.params.cat_ids
+ )
+ targets = self.coco_targets.get_annotations(annotation_ids)
+ # Get the prediction samples for the evaluation
+ prediction_ids = self.coco_predictions.get_annotation_ids(
+ img_ids=self.params.img_ids, cat_ids=self.params.cat_ids
+ )
+ predictions = self.coco_predictions.get_annotations(prediction_ids)
+
+ # Set ignore flag
+ for gt in targets:
+ gt["ignore"] = gt["ignore"] if "ignore" in gt else 0
+ gt["ignore"] = "iscrowd" in gt and gt["iscrowd"]
+
+ # Select targets
+ self._targets = defaultdict(list)
+ for gt in targets:
+ self._targets[gt["image_id"], gt["category_id"]].append(gt)
+
+ # Select predictions
+ self._predictions = defaultdict(list)
+ for dt in predictions:
+ self._predictions[dt["image_id"], dt["category_id"]].append(dt)
+
+ # Initialize evaluation results
+ self.eval_imgs = defaultdict(list)
+ self.results = {}
+
+ def _compute_iou(self, img_id: int, cat_id: int) -> np.ndarray:
+ """
+ Compute the IoU between the targets and predictions for a given image and
+ category.
+
+ Args:
+ img_id (int): The image id.
+ cat_id (int): The category id.
+
+ Returns:
+ np.ndarray: The IoU between the targets and predictions.
+ """
+
+ gt = self._targets[img_id, cat_id]
+ dt = self._predictions[img_id, cat_id]
+
+ # If there is nothing to evaluate
+ if len(gt) == 0 and len(dt) == 0:
+ return np.array([])
+
+ # Sort predictions by highest score first
+ inds = np.argsort([-d["score"] for d in dt], kind="stable")
+ dt = [dt[i] for i in inds]
+
+ # Truncate the predictions if there are more predictions than the max detections
+ # to evaluate
+ if len(dt) > self.params.max_dets[-1]:
+ dt = dt[0 : self.params.max_dets[-1]]
+
+ gt_boxes = [g["bbox"] for g in gt]
+ dt_boxes = [d["bbox"] for d in dt]
+
+ # Get the iscrowd flag for each gt
+ is_crowd = [int(o["iscrowd"]) for o in gt]
+ # Compute iou between each prediction a and gt region
+ iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd)
+ return iou
+
+ def _evaluate_image(
+ self, img_id: int, cat_id: int, area_range: Tuple[int, int], max_det: int
+ ) -> Union[Dict[str, Any], None]:
+ """
+ Perform evaluation for single category and image.
+ Args:
+ img_id (int): The image id.
+ cat_id (int): The category id.
+ area_range (Tuple[int, int]): The area range.
+ max_det (int): The maximum number of detections.
+
+ Returns:
+ Dict[str, Any]: The evaluation results.
+ """
+ # Get targets (gt) and predictions (dt) for the given image and category
+ gt = self._targets[img_id, cat_id]
+ dt = self._predictions[img_id, cat_id]
+
+ # If there is nothing to evaluate
+ if len(gt) == 0 and len(dt) == 0:
+ return None
+
+ min_area, max_area = area_range
+
+ # Create an `_ignore` flag for targets if they are set as ignore or their area
+ # is not in the range [min_area, max_area]
+ for g in gt:
+ if g["ignore"] or not (min_area <= g["area"] <= max_area):
+ g["_ignore"] = 1
+ else:
+ g["_ignore"] = 0
+
+ # Sort ground-truths by ignore flag (0: non ignored, 1: ignored)
+ gt_sorted = np.argsort([g["_ignore"] for g in gt], kind="stable")
+ gt = [gt[i] for i in gt_sorted]
+
+ # Sort predictions by scores in descending order
+ dt_sorted = np.argsort([-d["score"] for d in dt], kind="stable")
+ dt = [dt[i] for i in dt_sorted[0:max_det]]
+
+ # Load computed ious for the given image and category
+ ious = (
+ self.ious[img_id, cat_id][:, gt_sorted]
+ if len(self.ious[img_id, cat_id]) > 0
+ else self.ious[img_id, cat_id]
+ )
+
+ # Get the number of thresholds, ground truths and detections
+ num_thresholds = len(self.params.iou_thrs)
+ num_ground_truths = len(gt)
+ num_detections = len(dt)
+
+ # Initialize matches: 0 means no match
+ gt_matches = np.zeros((num_thresholds, num_ground_truths))
+ dt_matches = np.zeros((num_thresholds, num_detections))
+ # Initialize ignore flags: 0 means no ignore
+ gt_ignore = np.array([g["_ignore"] for g in gt])
+ dt_ignore = np.zeros((num_thresholds, num_detections))
+ if len(ious) != 0:
+ # Go through the iou thresholds
+ for tresh_idx, thresh in enumerate(self.params.iou_thrs):
+ # Go through the detections
+ for det_idx, det in enumerate(dt):
+ # Start the iou of the best match
+ iou_best_match = min([thresh, 1 - 1e-10])
+ # Set the best match index to -1 (unmatched)
+ best_match_idx = -1
+ # Go through the ground truths
+ for g_idx, g in enumerate(gt):
+ # If current gt is already matched, and not a crowd, continue
+ # if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd[g_idx]:
+ iscrowd = int(g.get("iscrowd"))
+ if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd:
+ continue
+ # Stop searching the ground truths
+ if (
+ best_match_idx > -1 # detection is matched to a gt
+ and gt_ignore[best_match_idx]
+ == 0 # matched gt is not ignored
+ and gt_ignore[g_idx] == 1 # current gt is ignored
+ ):
+ break
+
+ # A new best match was found
+ if ious[det_idx, g_idx] >= iou_best_match:
+ iou_best_match = ious[det_idx, g_idx]
+ best_match_idx = g_idx
+
+ # A best match was found
+ if best_match_idx != -1:
+ dt_ignore[tresh_idx, det_idx] = gt_ignore[best_match_idx]
+ dt_matches[tresh_idx, det_idx] = gt[best_match_idx]["id"]
+ gt_matches[tresh_idx, best_match_idx] = det["id"]
+
+ # Set unmatched detections outside of area range to ignore
+ area_range_mask = np.array(
+ [d["area"] < min_area or d["area"] > max_area for d in dt]
+ ).reshape((1, len(dt)))
+
+ # Update the ignore flags for detections
+ dt_ignore = np.logical_or(
+ dt_ignore,
+ np.logical_and(
+ dt_matches == 0, np.repeat(area_range_mask, num_thresholds, 0)
+ ),
+ )
+
+ return {
+ "image_id": img_id,
+ "category_id": cat_id,
+ "area_range": area_range,
+ "max_det": max_det,
+ "dt_ids": [d["id"] for d in dt],
+ "gt_ids": [g["id"] for g in gt],
+ "dtMatches": dt_matches,
+ "gtMatches": gt_matches,
+ "dtScores": [d["score"] for d in dt],
+ "gtIgnore": gt_ignore,
+ "dtIgnore": dt_ignore,
+ }
+
+ def __str__(self):
+ self.summarize()
+
+ def _accumulate(self):
+ """
+ Accumulate per image evaluation results and store the result in self.results
+ """
+ # Get the number of thresholds, categories, area ranges, and max detections
+ num_iou_thresholds = len(self.params.iou_thrs)
+ num_recall_thresholds = len(self.params.rec_thrs)
+ num_categories = len(self.params.cat_ids)
+ num_area_ranges = len(self.params.area_range)
+ num_max_detections = len(self.params.max_dets)
+ num_imgs = len(self.params.img_ids)
+
+ # Initialize precision, recall, and scores arrays
+ # -1 means absent categories
+ precision = -np.ones(
+ (
+ num_iou_thresholds,
+ num_recall_thresholds,
+ num_categories,
+ num_area_ranges,
+ num_max_detections,
+ )
+ )
+ recall = -np.ones(
+ (num_iou_thresholds, num_categories, num_area_ranges, num_max_detections)
+ )
+ scores = -np.ones(
+ (
+ num_iou_thresholds,
+ num_recall_thresholds,
+ num_categories,
+ num_area_ranges,
+ num_max_detections,
+ )
+ )
+
+ # Create sets for indexing
+ set_categories = set(self.params.cat_ids)
+ set_area_ranges = set(map(tuple, self.params.area_range))
+ set_max_detections = set(self.params.max_dets)
+ set_image_ids = set(self.params.img_ids)
+
+ # Select category indexes to evaluate
+ selected_category_ids = [
+ n for n, k in enumerate(self.params.cat_ids) if k in set_categories
+ ]
+ # Select max detections to evaluate
+ selected_max_detections = [
+ m for m in self.params.max_dets if m in set_max_detections
+ ]
+ # Select area ranges to evaluate
+ selected_area_ranges_ids = [
+ idx
+ for idx, area in enumerate(self.params.area_range)
+ if tuple(area) in set_area_ranges
+ ]
+ # Select image indexes to evaluate
+ image_inds = [
+ n for n, i in enumerate(self.params.img_ids) if i in set_image_ids
+ ]
+
+ # Evaluating at all categories, area ranges, max number of detections, and
+ # IoU thresholds
+
+ # Loop through categories
+ for cat_idx, cat_eval_idx in enumerate(selected_category_ids):
+ cat_offset = cat_eval_idx * num_area_ranges * num_imgs
+
+ # Loop through area ranges
+ for area_idx, area_eval_idx in enumerate(selected_area_ranges_ids):
+ area_offset = area_eval_idx * num_imgs
+
+ # Loop through max detections
+ for max_det_idx, max_det in enumerate(selected_max_detections):
+ eval_img_data = [
+ self.eval_imgs[cat_offset + area_offset + i] for i in image_inds
+ ]
+ eval_img_data = [e for e in eval_img_data if e is not None]
+
+ # No image to evaluate
+ if len(eval_img_data) == 0:
+ continue
+
+ # Sort detected scores in descending order
+ dt_scores = np.concatenate(
+ [e["dtScores"][0:max_det] for e in eval_img_data]
+ )
+ inds = np.argsort(-dt_scores, kind="stable")
+ dt_scores_sorted = dt_scores[inds]
+
+ # Get matches and ignored matches
+ dt_matches = np.concatenate(
+ [e["dtMatches"][:, 0:max_det] for e in eval_img_data], axis=1
+ )[:, inds]
+ dt_ignored = np.concatenate(
+ [e["dtIgnore"][:, 0:max_det] for e in eval_img_data], axis=1
+ )[:, inds]
+
+ # Get ignored ground truth objects
+ gt_ignored = np.concatenate([e["gtIgnore"] for e in eval_img_data])
+ num_non_ignored_gt = np.count_nonzero(gt_ignored == 0)
+
+ # No ground truth objects to evaluate
+ if num_non_ignored_gt == 0:
+ continue
+
+ # Compute true positives and false positives
+ true_positives = np.logical_and(
+ dt_matches, np.logical_not(dt_ignored)
+ )
+ false_positives = np.logical_and(
+ np.logical_not(dt_matches), np.logical_not(dt_ignored)
+ )
+
+ tp_sum = np.cumsum(true_positives, axis=1).astype(dtype=np.float64)
+ fp_sum = np.cumsum(false_positives, axis=1).astype(dtype=np.float64)
+
+ # Loop through thresholds
+ for iou_thresh_idx, (tp, fp) in enumerate(zip(tp_sum, fp_sum)):
+ tp = np.array(tp)
+ fp = np.array(fp)
+ num_tps = len(tp)
+ # Recall: TP / Total number of ground truth objects
+ rc = tp / num_non_ignored_gt
+ # Precision: TP / (FP + TP)
+ pr = (tp / (fp + tp + EPS)).tolist()
+ # List to compute the precision at each recall threshold
+ precision_at_recall = [0] * num_recall_thresholds
+ # List to compute the score at each recall threshold
+ score_at_recall = [0] * num_recall_thresholds
+
+ # Set recall to either the final recall value or 0 (when there
+ # is no TP)
+ recall[iou_thresh_idx, cat_idx, area_idx, max_det_idx] = (
+ rc[-1] if num_tps else 0
+ )
+
+ # Loop through precision values
+ for i in range(num_tps - 1, 0, -1):
+ if pr[i] > pr[i - 1]:
+ pr[i - 1] = pr[i]
+
+ inds = np.searchsorted(rc, self.params.rec_thrs, side="left")
+ for ri, pos_idx in enumerate(inds):
+ # Ensure pi is within the range of both arrays
+ if 0 <= pos_idx < len(pr) and 0 <= pos_idx < len(
+ dt_scores_sorted
+ ):
+ precision_at_recall[ri] = pr[pos_idx]
+ score_at_recall[ri] = dt_scores_sorted[pos_idx]
+
+ # Convert precision to numpy array
+ precision[iou_thresh_idx, :, cat_idx, area_idx, max_det_idx] = (
+ np.array(precision_at_recall)
+ )
+ # Convert scores to numpy array
+ scores[iou_thresh_idx, :, cat_idx, area_idx, max_det_idx] = (
+ np.array(score_at_recall)
+ )
+
+ # Average precision over all sizes, 100 max detections
+ area_range_idx = list(ObjectSize).index(ObjectSize.ALL)
+ max_100_dets_idx = self.params.max_dets.index(100)
+ # Average precision [threshold, recall, classes]
+ average_precision_all_sizes = precision[
+ :, :, :, area_range_idx, max_100_dets_idx
+ ]
+ # mAP over thresholds (dimension=num_thresholds)
+ mAP_scores_all_sizes = average_precision_all_sizes.mean(axis=(1, 2))
+ # AP per class
+ ap_per_class_all_sizes = average_precision_all_sizes.mean(axis=1).transpose(
+ 1, 0
+ )
+
+ # Average precision for SMALL objects and 100 max detections
+ small_area_range_idx = list(ObjectSize).index(ObjectSize.SMALL)
+ average_precision_small = precision[
+ :, :, :, small_area_range_idx, max_100_dets_idx
+ ]
+ mAP_scores_small = average_precision_small.mean(axis=(1, 2))
+ ap_per_class_small = average_precision_small.mean(axis=1).transpose(1, 0)
+
+ # Average precision for MEDIUM objects and 100 max detections
+ medium_area_range_idx = list(ObjectSize).index(ObjectSize.MEDIUM)
+ average_precision_medium = precision[
+ :, :, :, medium_area_range_idx, max_100_dets_idx
+ ]
+ mAP_scores_medium = average_precision_medium.mean(axis=(1, 2))
+ ap_per_class_medium = average_precision_medium.mean(axis=1).transpose(1, 0)
+
+ # Average precision for LARGE objects and 100 max detections
+ large_area_range_idx = list(ObjectSize).index(ObjectSize.LARGE)
+ average_precision_large = precision[
+ :, :, :, large_area_range_idx, max_100_dets_idx
+ ]
+ mAP_scores_large = average_precision_large.mean(axis=(1, 2))
+ ap_per_class_large = average_precision_large.mean(axis=1).transpose(1, 0)
+
+ self.results = {
+ "params": self.params,
+ "counts": [
+ num_iou_thresholds,
+ num_recall_thresholds,
+ num_categories,
+ num_area_ranges,
+ num_max_detections,
+ ],
+ "date": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+ "precision": precision,
+ "recall": recall,
+ "scores": scores,
+ "mAP_scores_all_sizes": mAP_scores_all_sizes,
+ "ap_per_class_all_sizes": ap_per_class_all_sizes,
+ "mAP_scores_small": mAP_scores_small,
+ "ap_per_class_small": ap_per_class_small,
+ "mAP_scores_medium": mAP_scores_medium,
+ "ap_per_class_medium": ap_per_class_medium,
+ "mAP_scores_large": mAP_scores_large,
+ "ap_per_class_large": ap_per_class_large,
+ }
+
+ def _pycocotools_summarize(self):
+ """
+ Compute and display summary metrics for evaluation results.
+ """
+
+ def _summarize(
+ use_ap: bool = True, iou_thr=None, area_range=ObjectSize.ALL, max_dets=100
+ ):
+ iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.10f}"
+ titleStr = "Average Precision" if use_ap else "Average Recall"
+ typeStr = "(AP)" if use_ap else "(AR)"
+ iou_str = (
+ "{:0.2f}:{:0.2f}".format(
+ self.params.iou_thrs[0], self.params.iou_thrs[-1]
+ )
+ if iou_thr is None
+ else "{:0.2f}".format(iou_thr)
+ )
+ all_object_sizes = list(ObjectSize)
+ area_range_idx = all_object_sizes.index(area_range)
+ max_detections_idx = self.params.max_dets.index(max_dets)
+ if use_ap:
+ # Dimension of precision:
+ # threshold x recall x classes x areas x max detections
+ s = self.results["precision"]
+ # IOU
+ if iou_thr is not None:
+ t = np.where(iou_thr == self.params.iou_thrs)[0]
+ s = s[t]
+ s = s[:, :, :, area_range_idx, max_detections_idx]
+ else:
+ # Dimension of recall:
+ # threshold x classes x areas x max detections
+ s = self.results["recall"]
+ if iou_thr is not None:
+ t = np.where(iou_thr == self.params.iou_thrs)[0]
+ s = s[t]
+ s = s[:, :, area_range_idx, max_detections_idx]
+ if len(s[s > -1]) == 0:
+ mean_s = -1
+ else:
+ mean_s = np.mean(s[s > -1])
+ print(iStr.format(titleStr, typeStr, iou_str, area_range, max_dets, mean_s))
+ return mean_s
+
+ def _summarize_predictions():
+ stats = np.zeros((12,))
+ stats[0] = _summarize(use_ap=True)
+ stats[1] = _summarize(
+ use_ap=True, iou_thr=0.5, max_dets=self.params.max_dets[2]
+ )
+ stats[2] = _summarize(
+ use_ap=True, iou_thr=0.75, max_dets=self.params.max_dets[2]
+ )
+ stats[3] = _summarize(
+ use_ap=True,
+ area_range=ObjectSize.SMALL,
+ max_dets=self.params.max_dets[2],
+ )
+ stats[4] = _summarize(
+ use_ap=True,
+ area_range=ObjectSize.MEDIUM,
+ max_dets=self.params.max_dets[2],
+ )
+ stats[5] = _summarize(
+ use_ap=True,
+ area_range=ObjectSize.LARGE,
+ max_dets=self.params.max_dets[2],
+ )
+ stats[6] = _summarize(use_ap=False, max_dets=self.params.max_dets[0])
+ stats[7] = _summarize(use_ap=False, max_dets=self.params.max_dets[1])
+ stats[8] = _summarize(use_ap=False, max_dets=self.params.max_dets[2])
+ stats[9] = _summarize(
+ use_ap=False,
+ area_range=ObjectSize.SMALL,
+ max_dets=self.params.max_dets[2],
+ )
+ stats[10] = _summarize(
+ use_ap=False,
+ area_range=ObjectSize.MEDIUM,
+ max_dets=self.params.max_dets[2],
+ )
+ stats[11] = _summarize(
+ use_ap=False,
+ area_range=ObjectSize.LARGE,
+ max_dets=self.params.max_dets[2],
+ )
+ return stats
+
+ if len(self.results) != 0:
+ self.stats = _summarize_predictions()
+
+ def evaluate(self):
+ """
+ Start the per image evaluation on all images and keeep results in
+ self.eval_imgs (a list of dictionaries).
+ """
+ # Select all parameters to evaluate
+ self.params.img_ids = list(np.unique(self.params.img_ids))
+ self.params.cat_ids = list(np.unique(self.params.cat_ids))
+ self.params.max_dets = sorted(self.params.max_dets)
+
+ self._prepare_targets_and_predictions()
+
+ # Compute IOUs between all targets and predictions for all images and categories
+ self.ious = {
+ (img_id, cat_id): self._compute_iou(img_id, cat_id)
+ for img_id in self.params.img_ids
+ for cat_id in self.params.cat_ids
+ }
+
+ # Select the largest max area (the last element containing 100 dets
+ max_det = self.params.max_dets[-1]
+
+ # Evaluate each image with all categories, area range and max detections
+ self.eval_imgs = [
+ self._evaluate_image(img_id, cat_id, area_range, max_det)
+ for cat_id in self.params.cat_ids
+ for area_range in self.params.area_range
+ for img_id in self.params.img_ids
+ ]
+
+ # Accumulate results
+ self._accumulate()
+
+
+class MeanAveragePrecision(Metric):
+ """
+ Mean Average Precision (mAP) is a metric used to evaluate object detection models.
+ It is the average of the precision-recall curves at different IoU thresholds.
+
+ Example:
+ ```python
+ import supervision as sv
+ from supervision.metrics import MeanAveragePrecision
+
+ predictions = sv.Detections(...)
+ targets = sv.Detections(...)
+
+ map_metric = MeanAveragePrecision()
+ map_result = map_metric.update(predictions, targets).compute()
+
+ print(map_result.map50_95)
+ # 0.4674
+
+ print(map_result)
+ # MeanAveragePrecisionResult:
+ # Metric target: MetricTarget.BOXES
+ # Class agnostic: False
+ # mAP @ 50:95: 0.4674
+ # mAP @ 50: 0.5048
+ # mAP @ 75: 0.4796
+ # mAP scores: [0.50485 0.50377 0.50377 ...]
+ # IoU thresh: [0.5 0.55 0.6 ...]
+ # AP per class:
+ # 0: [0.67699 0.67699 0.67699 ...]
+ # ...
+ # Small objects: ...
+ # Medium objects: ...
+ # Large objects: ...
+
+ map_result.plot()
+ ```
+
+ { align=center width="800" }
+ """
+
+ def __init__(
+ self,
+ metric_target: MetricTarget = MetricTarget.BOXES,
+ class_agnostic: bool = False,
+ class_mapping: Optional[Dict[int, int]] = None,
+ image_indices: Optional[List[int]] = None,
+ ):
+ """
+ Initialize the Mean Average Precision metric.
+
+ Args:
+ metric_target (MetricTarget): The type of detection data to use.
+ class_agnostic (bool): Whether to treat all data as a single class.
+ class_mapping (Optional[Dict[int, int]]): A dictionary to map class IDs to
+ new IDs.
+ image_indices (Optional[List[int]]): The indices of the images to use.
+ """
+ self._metric_target = metric_target
+ self._class_agnostic = class_agnostic
+
+ self._predictions_list: List[Detections] = []
+ self._targets_list: List[Detections] = []
+ self._class_mapping = class_mapping
+ self._image_indices = image_indices
+
+ def reset(self) -> None:
+ """
+ Reset the metric to its initial state, clearing all stored data.
+ """
+ self._predictions_list = []
+ self._targets_list = []
+
+ def update(
+ self,
+ predictions: Union[Detections, List[Detections]],
+ targets: Union[Detections, List[Detections]],
+ ) -> MeanAveragePrecision:
+ """
+ Add new predictions and targets to the metric, but do not compute the result.
+
+ Args:
+ predictions (Union[Detections, List[Detections]]): The predicted detections.
+ targets (Union[Detections, List[Detections]]): The ground-truth detections.
+
+ Returns:
+ (MeanAveragePrecision): The updated metric instance.
+ """
+ if not isinstance(predictions, list):
+ predictions = [predictions]
+ if not isinstance(targets, list):
+ targets = [targets]
+
+ if len(predictions) != len(targets):
+ raise ValueError(
+ f"The number of predictions ({len(predictions)}) and"
+ f" targets ({len(targets)}) during the update must be the same."
+ )
+
+ if self._class_agnostic:
+ predictions = deepcopy(predictions)
+ targets = deepcopy(targets)
+
+ for prediction in predictions:
+ prediction.class_id[:] = -1
+ for target in targets:
+ target.class_id[:] = -1
+
+ self._predictions_list.extend(predictions)
+ self._targets_list.extend(targets)
+
+ return self
+
+ def _prepare_targets(self, targets):
+ """Transform targets into a dictionary that can be used by the COCO evaluator"""
+ images = [{"id": img_id} for img_id in range(len(targets))]
+ if self._image_indices is not None:
+ images = [
+ {"id": self._image_indices[img_id.get("id")]} for img_id in images
+ ]
+ # Annotations list
+ annotations = []
+ for image_id, image_targets in enumerate(targets):
+ if self._image_indices is not None:
+ image_id = self._image_indices[image_id]
+ for target in image_targets:
+ xyxy = target[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2]
+ xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]]
+ # Get "area" and "iscrowd" (default 0) from data
+ data = target[5]
+
+ if self._class_mapping is not None:
+ category_id = self._class_mapping[target[3].item()]
+ else:
+ category_id = target[3].item()
+ dict_annotation = {
+ "area": data.get("area", 0),
+ "iscrowd": data.get("iscrowd", 0),
+ "image_id": image_id,
+ "bbox": xywh,
+ "category_id": category_id,
+ "id": len(annotations), # incrementally increase the id
+ }
+ annotations.append(dict_annotation)
+ # Category list
+ all_cat_ids = set([annotation.get("category_id") for annotation in annotations])
+ categories = [{"id": cat_id} for cat_id in all_cat_ids]
+ # Create coco dictionary
+ return {
+ "images": images,
+ "annotations": annotations,
+ "categories": categories,
+ }
+
+ def _prepare_predictions(self, predictions):
+ """Transform predictions into a list of predictions that can be used by the COCO
+ evaluator."""
+ coco_predictions = []
+ for image_id, image_predictions in enumerate(predictions):
+ if self._image_indices is not None:
+ image_id = self._image_indices[image_id]
+ for prediction in image_predictions:
+ xyxy = prediction[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2]
+ xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]]
+ if self._class_mapping is not None:
+ category_id = self._class_mapping[prediction[3].item()]
+ else:
+ category_id = prediction[3].item()
+ dict_prediction = {
+ "image_id": image_id,
+ "bbox": xywh,
+ "score": prediction[2].item(),
+ "category_id": category_id,
+ }
+ coco_predictions.append(dict_prediction)
+ return coco_predictions
+
+ def compute(self) -> MeanAveragePrecisionResult:
+ """
+ Calculate Mean Average Precision based on predicted and ground-truth
+ detections at different thresholds using the COCO evaluation metrics.
+ Source: https://github.com/rafaelpadilla/review_object_detection_metrics
+
+ Returns:
+ (MeanAveragePrecisionResult): The Mean Average Precision result.
+ """
+ total_images_predictions = len(self._predictions_list)
+ total_images_targets = len(self._targets_list)
+
+ if total_images_predictions != total_images_targets:
+ raise ValueError(
+ f"The number of predictions ({total_images_predictions}) and"
+ f" targets ({total_images_targets}) during the evaluation must be"
+ " the same."
+ )
+ dict_targets = self._prepare_targets(self._targets_list)
+ lst_predictions = self._prepare_predictions(self._predictions_list)
+ # Create a coco object with the targets
+ coco_gt = EvaluationDataset(targets=dict_targets)
+ # Include the predictions to coco object
+ coco_det = coco_gt.load_predictions(lst_predictions)
+ # Create a coco evaluator with the predictions
+ cocoEval = COCOEvaluator(coco_gt, coco_det)
+
+ # Evaluate on all images
+ cocoEval.evaluate()
+
+ # Create MeanAveragePrecisionResult object for small objects
+ mAP_small = MeanAveragePrecisionResult(
+ metric_target=self._metric_target,
+ is_class_agnostic=self._class_agnostic,
+ mAP_scores=cocoEval.results["mAP_scores_small"],
+ ap_per_class=cocoEval.results["ap_per_class_small"],
+ iou_thresholds=cocoEval.params.iou_thrs,
+ matched_classes=cocoEval.params.cat_ids,
+ )
+ # Create MeanAveragePrecisionResult object for medium objects
+ mAP_medium = MeanAveragePrecisionResult(
+ metric_target=self._metric_target,
+ is_class_agnostic=self._class_agnostic,
+ mAP_scores=cocoEval.results["mAP_scores_medium"],
+ ap_per_class=cocoEval.results["ap_per_class_medium"],
+ iou_thresholds=cocoEval.params.iou_thrs,
+ matched_classes=cocoEval.params.cat_ids,
+ )
+ # Create MeanAveragePrecisionResult object for large objects
+ mAP_large = MeanAveragePrecisionResult(
+ metric_target=self._metric_target,
+ is_class_agnostic=self._class_agnostic,
+ mAP_scores=cocoEval.results["mAP_scores_large"],
+ ap_per_class=cocoEval.results["ap_per_class_large"],
+ iou_thresholds=cocoEval.params.iou_thrs,
+ matched_classes=cocoEval.params.cat_ids,
+ )
+
+ # Create the final MeanAveragePrecisionResult object
+ mAP_result = MeanAveragePrecisionResult(
+ metric_target=self._metric_target,
+ is_class_agnostic=self._class_agnostic,
+ mAP_scores=cocoEval.results["mAP_scores_all_sizes"],
+ ap_per_class=cocoEval.results["ap_per_class_all_sizes"],
+ iou_thresholds=cocoEval.params.iou_thrs,
+ matched_classes=cocoEval.params.cat_ids,
+ small_objects=mAP_small,
+ medium_objects=mAP_medium,
+ large_objects=mAP_large,
+ )
+ return mAP_result
diff --git a/test/dataset/formats/test_coco.py b/test/dataset/formats/test_coco.py
index 3a68894a..b8f53461 100644
--- a/test/dataset/formats/test_coco.py
+++ b/test/dataset/formats/test_coco.py
@@ -162,12 +162,22 @@ def test_group_coco_annotations_by_image_id(
@pytest.mark.parametrize(
- "image_annotations, resolution_wh, with_masks, expected_result, exception",
+ "image_annotations, resolution_wh, with_masks, use_iscrowd, "
+ "expected_result, exception",
[
(
[],
(1000, 1000),
False,
+ False,
+ Detections.empty(),
+ DoesNotRaise(),
+ ), # empty image annotations
+ (
+ [],
+ (1000, 1000),
+ False,
+ True,
Detections.empty(),
DoesNotRaise(),
), # empty image annotations
@@ -179,12 +189,32 @@ def test_group_coco_annotations_by_image_id(
],
(1000, 1000),
False,
+ False,
Detections(
xyxy=np.array([[0, 0, 100, 100]], dtype=np.float32),
class_id=np.array([0], dtype=int),
),
DoesNotRaise(),
), # single image annotations
+ (
+ [
+ mock_coco_annotation(
+ category_id=0, bbox=(0, 0, 100, 100), area=100 * 100
+ )
+ ],
+ (1000, 1000),
+ False,
+ True,
+ Detections(
+ xyxy=np.array([[0, 0, 100, 100]], dtype=np.float32),
+ class_id=np.array([0], dtype=int),
+ data={
+ "iscrowd": np.array([0], dtype=int),
+ "area": np.array([100 * 100]),
+ },
+ ),
+ DoesNotRaise(),
+ ),
(
[
mock_coco_annotation(
@@ -196,6 +226,7 @@ def test_group_coco_annotations_by_image_id(
],
(1000, 1000),
False,
+ False,
Detections(
xyxy=np.array(
[[0, 0, 100, 100], [100, 100, 200, 200]], dtype=np.float32
@@ -204,6 +235,30 @@ def test_group_coco_annotations_by_image_id(
),
DoesNotRaise(),
), # two image annotations
+ (
+ [
+ mock_coco_annotation(
+ category_id=0, bbox=(0, 0, 100, 100), area=100 * 100
+ ),
+ mock_coco_annotation(
+ category_id=0, bbox=(100, 100, 100, 100), area=100 * 100
+ ),
+ ],
+ (1000, 1000),
+ False,
+ True,
+ Detections(
+ xyxy=np.array(
+ [[0, 0, 100, 100], [100, 100, 200, 200]], dtype=np.float32
+ ),
+ class_id=np.array([0, 0], dtype=int),
+ data={
+ "iscrowd": np.array([0, 0], dtype=int),
+ "area": np.array([100 * 100, 100 * 100]),
+ },
+ ),
+ DoesNotRaise(),
+ ),
(
[
mock_coco_annotation(
@@ -215,6 +270,7 @@ def test_group_coco_annotations_by_image_id(
],
(5, 5),
True,
+ False,
Detections(
xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32),
class_id=np.array([0], dtype=int),
@@ -232,6 +288,36 @@ def test_group_coco_annotations_by_image_id(
),
DoesNotRaise(),
), # single image annotations with mask as polygon
+ (
+ [
+ mock_coco_annotation(
+ category_id=0,
+ bbox=(0, 0, 5, 5),
+ area=5 * 5,
+ segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]],
+ )
+ ],
+ (5, 5),
+ True,
+ True,
+ Detections(
+ xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32),
+ class_id=np.array([0], dtype=int),
+ mask=np.array(
+ [
+ [
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ ]
+ ]
+ ),
+ data={"iscrowd": np.array([0], dtype=int), "area": np.array([25])},
+ ),
+ DoesNotRaise(),
+ ),
(
[
mock_coco_annotation(
@@ -247,6 +333,7 @@ def test_group_coco_annotations_by_image_id(
],
(5, 5),
True,
+ False,
Detections(
xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32),
class_id=np.array([0], dtype=int),
@@ -264,6 +351,40 @@ def test_group_coco_annotations_by_image_id(
),
DoesNotRaise(),
), # single image annotations with mask, RLE segmentation mask
+ (
+ [
+ mock_coco_annotation(
+ category_id=0,
+ bbox=(0, 0, 5, 5),
+ area=5 * 5,
+ segmentation={
+ "size": [5, 5],
+ "counts": [0, 15, 2, 3, 2, 3],
+ },
+ iscrowd=True,
+ )
+ ],
+ (5, 5),
+ True,
+ True,
+ Detections(
+ xyxy=np.array([[0, 0, 5, 5]], dtype=np.float32),
+ class_id=np.array([0], dtype=int),
+ mask=np.array(
+ [
+ [
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ ]
+ ]
+ ),
+ data={"iscrowd": np.array([1], dtype=int), "area": np.array([25])},
+ ),
+ DoesNotRaise(),
+ ),
(
[
mock_coco_annotation(
@@ -285,6 +406,7 @@ def test_group_coco_annotations_by_image_id(
],
(5, 5),
True,
+ False,
Detections(
xyxy=np.array([[0, 0, 5, 5], [3, 0, 5, 2]], dtype=np.float32),
class_id=np.array([0, 0], dtype=int),
@@ -309,6 +431,57 @@ def test_group_coco_annotations_by_image_id(
),
DoesNotRaise(),
), # two image annotations with mask, one mask as polygon and second as RLE
+ (
+ [
+ mock_coco_annotation(
+ category_id=0,
+ bbox=(0, 0, 5, 5),
+ area=5 * 5,
+ segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]],
+ ),
+ mock_coco_annotation(
+ category_id=0,
+ bbox=(3, 0, 2, 2),
+ area=2 * 2,
+ segmentation={
+ "size": [5, 5],
+ "counts": [15, 2, 3, 2, 3],
+ },
+ iscrowd=True,
+ ),
+ ],
+ (5, 5),
+ True,
+ True,
+ Detections(
+ xyxy=np.array([[0, 0, 5, 5], [3, 0, 5, 2]], dtype=np.float32),
+ class_id=np.array([0, 0], dtype=int),
+ mask=np.array(
+ [
+ [
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ ],
+ [
+ [0, 0, 0, 1, 1],
+ [0, 0, 0, 1, 1],
+ [0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0],
+ ],
+ ]
+ ),
+ data={
+ "iscrowd": np.array([0, 1], dtype=int),
+ "area": np.array([25, 4]),
+ },
+ ),
+ DoesNotRaise(),
+ ), # two image annotations with mask, one mask as polygon with iscrowd,
+ # and second as RLE without iscrowd
(
[
mock_coco_annotation(
@@ -330,6 +503,7 @@ def test_group_coco_annotations_by_image_id(
],
(5, 5),
True,
+ False,
Detections(
xyxy=np.array([[3, 0, 5, 2], [0, 0, 5, 5]], dtype=np.float32),
class_id=np.array([0, 1], dtype=int),
@@ -354,12 +528,64 @@ def test_group_coco_annotations_by_image_id(
),
DoesNotRaise(),
), # two image annotations with mask, first mask as RLE and second as polygon
+ (
+ [
+ mock_coco_annotation(
+ category_id=0,
+ bbox=(3, 0, 2, 2),
+ area=2 * 2,
+ segmentation={
+ "size": [5, 5],
+ "counts": [15, 2, 3, 2, 3],
+ },
+ iscrowd=True,
+ ),
+ mock_coco_annotation(
+ category_id=1,
+ bbox=(0, 0, 5, 5),
+ area=5 * 5,
+ segmentation=[[0, 0, 2, 0, 2, 2, 4, 2, 4, 4, 0, 4]],
+ ),
+ ],
+ (5, 5),
+ True,
+ True,
+ Detections(
+ xyxy=np.array([[3, 0, 5, 2], [0, 0, 5, 5]], dtype=np.float32),
+ class_id=np.array([0, 1], dtype=int),
+ mask=np.array(
+ [
+ [
+ [0, 0, 0, 1, 1],
+ [0, 0, 0, 1, 1],
+ [0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0],
+ [0, 0, 0, 0, 0],
+ ],
+ [
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 0, 0],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ [1, 1, 1, 1, 1],
+ ],
+ ]
+ ),
+ data={
+ "iscrowd": np.array([1, 0], dtype=int),
+ "area": np.array([4, 25]),
+ },
+ ),
+ DoesNotRaise(),
+ ), # two image annotations with mask, first mask as RLE with is crowd,
+ # and second as polygon without iscrowd
],
)
def test_coco_annotations_to_detections(
image_annotations: List[dict],
resolution_wh: Tuple[int, int],
with_masks: bool,
+ use_iscrowd: bool,
expected_result: Detections,
exception: Exception,
) -> None:
@@ -368,6 +594,7 @@ def test_coco_annotations_to_detections(
image_annotations=image_annotations,
resolution_wh=resolution_wh,
with_masks=with_masks,
+ use_iscrowd=use_iscrowd,
)
assert result == expected_result