diff --git a/docs/how_to/count_in_zone.md b/docs/how_to/count_in_zone.md index 99c8b87c..8159f1f0 100644 --- a/docs/how_to/count_in_zone.md +++ b/docs/how_to/count_in_zone.md @@ -24,7 +24,7 @@ download_assets(VideoAssets.VEHICLES_2) First, we need to initialize a model. Let's use a YOLOv8 model with the default COCO checkpoint. We also need to load a video on which to run inference. -Create a YOLO model instance and load the source video using supervision's `VideoInfo` helper. The model will process each frame during inference, while `VideoInfo` extracts resolution and frame-rate metadata needed by the polygon zone annotator. A shared color palette ensures consistent zone coloring throughout the output video. +Create a YOLO model instance and download the source video. The model will process each frame during inference. A shared color palette ensures consistent zone coloring throughout the output video. ```python import numpy as np @@ -32,13 +32,13 @@ import supervision as sv import cv2 from ultralytics import YOLO +from supervision.assets import VideoAssets, download_assets model = YOLO("yolov8s.pt") -VIDEO = str(VideoAssets.VEHICLES_2) +VIDEO = download_assets(VideoAssets.VEHICLES_2) -colors = sv.ColorPalette.default() -video_info = sv.VideoInfo.from_video_path(VIDEO) +colors = sv.ColorPalette.DEFAULT ``` ## Calculate Coordinates @@ -80,10 +80,7 @@ With the coordinates of the zones to draw ready, we can set up our zones: Instantiate a `PolygonZone` for each polygon array, pairing it with a `PolygonZoneAnnotator` for visual overlay and a `BoxAnnotator` for drawing detection boxes. Each zone will later trigger on incoming detections to determine which objects fall inside its boundaries, enabling per-zone counting in the inference callback. ```python -zones = [ - sv.PolygonZone(polygon=polygon, frame_resolution_wh=video_info.resolution_wh) - for polygon in polygons -] +zones = [sv.PolygonZone(polygon=polygon) for polygon in polygons] zone_annotators = [ sv.PolygonZoneAnnotator( zone=zone, @@ -98,8 +95,6 @@ box_annotators = [ sv.BoxAnnotator( color=colors.by_idx(index), thickness=4, - text_thickness=4, - text_scale=2, ) for index in range(len(polygons)) ] @@ -121,9 +116,7 @@ def process_frame(frame: np.ndarray, i) -> np.ndarray: ): mask = zone.trigger(detections=detections) detections_filtered = detections[mask] - frame = box_annotator.annotate( - scene=frame, detections=detections_filtered, skip_label=True - ) + frame = box_annotator.annotate(scene=frame, detections=detections_filtered) frame = zone_annotator.annotate(scene=frame) return frame diff --git a/src/supervision/dataset/core.py b/src/supervision/dataset/core.py index 4e94cb1b..f74920de 100644 --- a/src/supervision/dataset/core.py +++ b/src/supervision/dataset/core.py @@ -72,9 +72,11 @@ class DetectionDataset(BaseDataset): Attributes: classes: List containing dataset class names. images: - Accepts a list of image paths, or dictionaries of loaded cv2 images - with paths as keys. If you pass a list of paths, the dataset will - lazily load images on demand, which is much more memory-efficient. + Accepts a list of image paths. Passing a dict + (``Dict[str, np.ndarray]``) is deprecated in ``0.30.0`` and will + be removed in ``0.33.0``; use a list of paths instead. + When a list of paths is provided, images are loaded lazily on + demand, which is more memory-efficient. annotations: Dictionary mapping image path to annotations. The dictionary keys match match the keys in `images` or entries in the list of @@ -107,6 +109,13 @@ class DetectionDataset(BaseDataset): self.image_paths = list(dict.fromkeys(images)) self._images_in_memory: dict[str, npt.NDArray[np.uint8]] = {} + if isinstance(images, dict): + self._images_in_memory = images + warn_deprecated( + "Passing a `Dict[str, np.ndarray]` into `DetectionDataset` is " + "deprecated in `0.30.0` and will be removed in `0.33.0`. Use " + "a list of paths `List[str]` instead." + ) def _get_image(self, image_path: str) -> npt.NDArray[np.uint8]: """Assumes that image is in dataset.""" diff --git a/src/supervision/metrics/detection.py b/src/supervision/metrics/detection.py index f54e5c38..6b478fd7 100644 --- a/src/supervision/metrics/detection.py +++ b/src/supervision/metrics/detection.py @@ -1142,7 +1142,9 @@ class ConfusionMatrix: Confusion matrix plot. """ - array = self.matrix.copy() + # Cast to float so that the NaN masking below never hits an integer + # matrix (assigning NaN into an int array raises ValueError). + array = self.matrix.astype(np.float64) if normalize: eps = 1e-8 diff --git a/src/supervision/metrics/mean_average_precision.py b/src/supervision/metrics/mean_average_precision.py index 1b4e14a2..56064328 100644 --- a/src/supervision/metrics/mean_average_precision.py +++ b/src/supervision/metrics/mean_average_precision.py @@ -7,14 +7,19 @@ from collections import defaultdict from copy import deepcopy from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, TypeAlias, TypedDict +from typing import TYPE_CHECKING, Any, TypeAlias, TypedDict import numpy as np import numpy.typing as npt from matplotlib import pyplot as plt +from supervision.config import ORIENTED_BOX_COORDINATES from supervision.detection.core import Detections -from supervision.detection.utils.iou_and_nms import box_iou_batch_with_jaccard +from supervision.detection.utils.iou_and_nms import ( + box_iou_batch_with_jaccard, + mask_iou_batch, + oriented_box_iou_batch, +) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget from supervision.metrics.utils.utils import ensure_pandas_installed @@ -41,6 +46,11 @@ class _TypeCocoDict(TypedDict, total=False): supercategory: str caption: str keypoints: list[float] + # Metric-target-specific content: a boolean mask of shape (H, W) for + # `MetricTarget.MASKS` or an oriented box of shape (4, 2) for + # `MetricTarget.ORIENTED_BOUNDING_BOXES`. Absent for `MetricTarget.BOXES`. + # Invariant: shape/dtype match `metric_target` of the owning COCOEvaluator. + content: npt.NDArray[Any] _TypeCocoDataset: TypeAlias = dict[str, list[_TypeCocoDict]] @@ -569,6 +579,54 @@ MAX_ALL_OBJECT_AREA = 1e5**2 EPS = np.finfo(np.float32).eps +def _mask_iou_with_jaccard( + masks_true: list[npt.NDArray[np.bool_]], + masks_detection: list[npt.NDArray[np.bool_]], + is_crowd: list[bool], +) -> npt.NDArray[np.float64]: + """ + Calculate the IoU between detection masks (dt) and ground-truth masks (gt), + following the COCO convention: a detection may match any subregion of a + crowd ground truth, so for crowd rows the union collapses to the detection + area (mask counterpart of `box_iou_batch_with_jaccard`). + + Args: + masks_true: List of ground-truth masks of shape `(H, W)`. + masks_detection: List of detection masks of shape `(H, W)`. + is_crowd: List indicating if each ground-truth mask is a crowd region. + + Returns: + Array of IoU values of shape `(len(masks_detection), len(masks_true))`. + """ + if len(masks_detection) == 0 or len(masks_true) == 0: + return np.empty((len(masks_detection), len(masks_true)), dtype=np.float64) + + gt_masks = np.stack(masks_true).astype(bool) + dt_masks = np.stack(masks_detection).astype(bool) + crowd = np.asarray(is_crowd, dtype=bool) + + # Compute base IoU via the optimised path (float32 + memory-chunked). + # mask_iou_batch returns (gt, dt); the evaluator expects (dt, gt). + iou: npt.NDArray[np.float64] = mask_iou_batch(gt_masks, dt_masks).T.astype( + np.float64 + ) + + if not np.any(crowd): + return iou + + # Override crowd columns: COCO convention collapses the union to the + # detection area, so a small detection inside a large crowd region scores + # IoU ≈ 1. Recompute only the crowd columns to avoid a full float64 matmul. + eps = np.spacing(1) + crowd_idx = np.where(crowd)[0] + gt_flat = gt_masks[crowd_idx].reshape(len(crowd_idx), -1).astype(np.float32) + dt_flat = dt_masks.reshape(dt_masks.shape[0], -1).astype(np.float32) + area_inter = (dt_flat @ gt_flat.T).astype(np.float64) # (dt, N_crowd) + area_dt = dt_flat.sum(axis=1).astype(np.float64) # (dt,) + iou[:, crowd_idx] = area_inter / (area_dt[:, None] + eps) + return iou + + class ObjectSize(Enum): """ Enum for object size. @@ -624,7 +682,10 @@ class COCOEvaluator: """ def __init__( - self, coco_targets: EvaluationDataset, coco_predictions: EvaluationDataset + self, + coco_targets: EvaluationDataset, + coco_predictions: EvaluationDataset, + metric_target: MetricTarget = MetricTarget.BOXES, ) -> None: """ Constructor of COCOEvaluator object. @@ -632,6 +693,8 @@ class COCOEvaluator: Args: coco_targets: The dataset with the ground truths. coco_predictions: The dataset with the predictions. + metric_target: The type of detection data used to compute the IoU - + boxes, masks or oriented bounding boxes. """ if coco_targets is None: raise ValueError("coco_targets must be provided") @@ -640,6 +703,7 @@ class COCOEvaluator: self.coco_targets = coco_targets self.coco_predictions = coco_predictions + self.metric_target = metric_target # List of dictionaries containing the evaluation results # len(eval_imgs) = (categories) * (area_ranges) * (images) # For COCO 2017: len(eval_images) = 80 * 4 * 5000 = 1600000 @@ -700,7 +764,8 @@ class COCOEvaluator: def _compute_iou(self, img_id: int, cat_id: int) -> npt.NDArray[np.float32]: """ Compute the IoU between the targets and predictions for a given image and - category. + category, using boxes, masks or oriented bounding boxes depending on the + configured metric target. Args: img_id: The image id. @@ -727,16 +792,30 @@ class COCOEvaluator: if len(dt) > self.params.max_dets[-1]: dt = dt[0 : self.params.max_dets[-1]] - gt_boxes = [g["bbox"] for g in gt] - dt_boxes = [d["bbox"] for d in dt] - # Get the iscrowd flag for each gt is_crowd = [bool(o["iscrowd"]) for o in gt] - # Compute iou between each prediction a and gt region - iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd).astype( - np.float32 - ) - return iou + + # Compute iou between each prediction and gt region + if self.metric_target == MetricTarget.MASKS: + iou = _mask_iou_with_jaccard( + [g["content"] for g in gt], [d["content"] for d in dt], is_crowd + ) + elif self.metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + # Crowd regions are not supported for oriented boxes; the standard + # IoU is used for every ground truth. + if len(gt) == 0 or len(dt) == 0: + iou = np.empty((len(dt), len(gt)), dtype=np.float64) + else: + gt_obb = np.stack([g["content"] for g in gt]) + dt_obb = np.stack([d["content"] for d in dt]) + # oriented_box_iou_batch returns (gt, dt); + # the evaluator expects (dt, gt). + iou = oriented_box_iou_batch(gt_obb, dt_obb).T.astype(np.float64) + else: + gt_boxes = [g["bbox"] for g in gt] + dt_boxes = [d["bbox"] for d in dt] + iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd) + return iou.astype(np.float32) def _evaluate_image( self, @@ -1379,6 +1458,43 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): return self + def _detections_content(self, detections: Detections) -> npt.NDArray[Any] | None: + """Return per-detection masks or oriented boxes for the metric target, + or `None` for the box target and for empty detections.""" + if self._metric_target == MetricTarget.BOXES or len(detections) == 0: + return None + if self._metric_target == MetricTarget.MASKS: + if detections.mask is None: + raise ValueError( + "MeanAveragePrecision with `MetricTarget.MASKS` requires" + " masks on both predictions and targets." + ) + return np.asarray(detections.mask).astype(bool) + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + obb = detections.data.get(ORIENTED_BOX_COORDINATES) + if obb is None: + raise ValueError( + "MeanAveragePrecision with" + " `MetricTarget.ORIENTED_BOUNDING_BOXES` requires" + f" `{ORIENTED_BOX_COORDINATES}` in `data` on both" + " predictions and targets." + ) + return np.asarray(obb, dtype=np.float32).reshape(-1, 4, 2) + raise ValueError(f"Invalid metric target: {self._metric_target}") + + def _content_area( + self, xywh: list[float], content: npt.NDArray[Any] | None, idx: int + ) -> float: + """Compute the default annotation area for the metric target: bbox area + for boxes, pixel count for masks, polygon area for oriented boxes.""" + if content is None: + return float(xywh[2] * xywh[3]) + if self._metric_target == MetricTarget.MASKS: + return float(np.count_nonzero(content[idx])) + x, y = content[idx, :, 0], content[idx, :, 1] + # Shoelace formula + return float(0.5 * abs(np.sum(x * np.roll(y, -1) - np.roll(x, -1) * y))) + def _prepare_targets( self, targets: list[Detections] ) -> dict[str, list[_TypeCocoDict]]: @@ -1396,6 +1512,7 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): if image_targets.xyxy is None: continue + content = self._detections_content(image_targets) for target_idx, xyxy in enumerate(image_targets.xyxy): xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] @@ -1409,7 +1526,8 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): else: category_id = int(cls_id) - # Use area from data if available, otherwise calculate from bbox + # Use area from data if available, otherwise calculate from the + # metric target content (bbox, mask or oriented box) area = None if image_targets.data is not None and "area" in image_targets.data: area_data: npt.NDArray[np.float32] = np.asarray( @@ -1418,7 +1536,7 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): area = float(area_data[target_idx]) if area is None: - area = xywh[2] * xywh[3] + area = self._content_area(xywh, content, target_idx) iscrowd = 0 if image_targets.data is not None and "iscrowd" in image_targets.data: @@ -1436,6 +1554,8 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): "id": len(annotations) + 1, # Start IDs from 1 (0 means no match) "ignore": 0, } + if content is not None: + dict_annotation["content"] = content[target_idx] annotations.append(dict_annotation) # Category list all_cat_ids = {annotation["category_id"] for annotation in annotations} @@ -1460,6 +1580,7 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): if image_predictions.xyxy is None: continue + content = self._detections_content(image_predictions) for pred_idx, xyxy in enumerate(image_predictions.xyxy): xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]] @@ -1476,7 +1597,8 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): if image_predictions.confidence is not None: score = float(image_predictions.confidence[pred_idx]) - # Use area from data if available, otherwise calculate from bbox + # Use area from data if available, otherwise calculate from the + # metric target content (bbox, mask or oriented box) area = None if ( image_predictions.data is not None @@ -1488,7 +1610,7 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): area = float(area_data[pred_idx]) if area is None: - area = xywh[2] * xywh[3] + area = self._content_area(xywh, content, pred_idx) dict_prediction: _TypeCocoDict = { "image_id": image_id, @@ -1498,6 +1620,8 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): "area": area, "id": len(coco_predictions) + 1, } + if content is not None: + dict_prediction["content"] = content[pred_idx] coco_predictions.append(dict_prediction) return coco_predictions @@ -1526,7 +1650,7 @@ class MeanAveragePrecision(Metric[MeanAveragePrecisionResult]): # Include the predictions to coco object coco_det = coco_gt.load_predictions(lst_predictions) # Create a coco evaluator with the predictions - cocoEval = COCOEvaluator(coco_gt, coco_det) + cocoEval = COCOEvaluator(coco_gt, coco_det, metric_target=self._metric_target) # Evaluate on all images cocoEval.evaluate() diff --git a/tests/dataset/test_core.py b/tests/dataset/test_core.py index 33220110..2774af1a 100644 --- a/tests/dataset/test_core.py +++ b/tests/dataset/test_core.py @@ -2,13 +2,19 @@ from contextlib import ExitStack as DoesNotRaise from pathlib import Path import numpy as np +import numpy.typing as npt import pytest from supervision import DetectionDataset, Detections from supervision.config import CLASS_NAME_DATA_FIELD +from supervision.utils.internal import SupervisionWarnings from tests.helpers import _create_detections, create_yolo_dataset +def _create_image(fill_value: int) -> npt.NDArray[np.uint8]: + return np.full((4, 4, 3), fill_value, dtype=np.uint8) + + @pytest.mark.parametrize( ("dataset_list", "expected_result", "exception"), [ @@ -282,3 +288,98 @@ class TestClassNamePopulation: np.testing.assert_array_equal( annotation.data[CLASS_NAME_DATA_FIELD], expected_names ) + + +class TestDetectionDatasetInMemoryImages: + """Verify DetectionDataset keeps dict-provided images in memory (DAT-01).""" + + @staticmethod + def _build_dataset( + images: dict[str, npt.NDArray[np.uint8]], + ) -> DetectionDataset: + annotations = { + path: _create_detections(xyxy=[[0, 0, 10, 10]], class_id=[0]) + for path in images + } + return DetectionDataset(classes=["dog"], images=images, annotations=annotations) + + def test_getitem_returns_in_memory_image(self) -> None: + """Indexing a dict-constructed dataset returns the in-memory array.""" + image = _create_image(fill_value=7) + dataset = self._build_dataset({"imgX.jpg": image}) + + image_path, loaded_image, _ = dataset[0] + + assert image_path == "imgX.jpg" + np.testing.assert_array_equal(loaded_image, image) + + def test_len_counts_in_memory_images(self) -> None: + """`len` of a dict-constructed dataset equals the number of provided images.""" + images = { + "img1.jpg": _create_image(fill_value=1), + "img2.jpg": _create_image(fill_value=2), + } + + dataset = self._build_dataset(images) + + assert len(dataset) == 2 + + def test_merge_preserves_in_memory_pixel_access(self) -> None: + """Merging two in-memory datasets keeps pixel access via public __getitem__.""" + image_1 = _create_image(fill_value=10) + image_2 = _create_image(fill_value=20) + ds_1 = self._build_dataset({"img1.jpg": image_1}) + ds_2 = self._build_dataset({"img2.jpg": image_2}) + + merged = DetectionDataset.merge([ds_1, ds_2]) + + assert len(merged) == 2 + _, loaded_1, _ = merged[0] + _, loaded_2, _ = merged[1] + np.testing.assert_array_equal(loaded_1, image_1) + np.testing.assert_array_equal(loaded_2, image_2) + + def test_iteration_yields_in_memory_images(self) -> None: + """Iteration yields (path, image, annotation) with correct pixels.""" + images = { + "img1.jpg": _create_image(fill_value=1), + "img2.jpg": _create_image(fill_value=2), + } + dataset = self._build_dataset(images) + + entries = list(dataset) + + assert [path for path, _, _ in entries] == ["img1.jpg", "img2.jpg"] + for image_path, loaded_image, annotation in entries: + np.testing.assert_array_equal(loaded_image, images[image_path]) + assert annotation is dataset.annotations[image_path] + + def test_dict_input_emits_deprecation_warning(self) -> None: + """Passing a dict of images emits the SupervisionWarnings deprecation notice.""" + with pytest.warns(SupervisionWarnings, match="deprecated"): + self._build_dataset({"img1.jpg": _create_image(fill_value=3)}) + + def test_eq_reflexive_in_memory(self) -> None: + """In-memory dataset equals itself (reflexive __eq__ via pixel comparison).""" + images = { + "img1.jpg": _create_image(fill_value=1), + "img2.jpg": _create_image(fill_value=2), + } + dataset = self._build_dataset(images) + + assert dataset == dataset + + def test_eq_same_pixels_returns_true(self) -> None: + """Two in-memory datasets with identical images and annotations are equal.""" + images = {"img1.jpg": _create_image(fill_value=5)} + ds_a = self._build_dataset(images) + ds_b = self._build_dataset(dict(images)) + + assert ds_a == ds_b + + def test_eq_different_pixels_returns_false(self) -> None: + """In-memory datasets with different pixel data are not equal.""" + ds_a = self._build_dataset({"img1.jpg": _create_image(fill_value=1)}) + ds_b = self._build_dataset({"img1.jpg": _create_image(fill_value=2)}) + + assert ds_a != ds_b diff --git a/tests/metrics/test_detection.py b/tests/metrics/test_detection.py index 30c3c893..2a016e00 100644 --- a/tests/metrics/test_detection.py +++ b/tests/metrics/test_detection.py @@ -4,6 +4,7 @@ from typing import ClassVar import cv2 import numpy as np import pytest +from matplotlib import pyplot as plt from supervision.dataset.core import DetectionDataset from supervision.detection.core import Detections @@ -1751,3 +1752,44 @@ class TestSplitDetectionsByOutcome: """Missing class_id on either input raises ValueError.""" with pytest.raises(ValueError, match="class_id"): _split_detections_by_outcome(predictions, targets, 0.5, 0.5) + + +class TestConfusionMatrixPlot: + """Tests for ConfusionMatrix.plot rendering.""" + + @pytest.mark.parametrize( + "normalize", + [ + pytest.param(False, id="raw-counts"), + pytest.param(True, id="normalized"), + ], + ) + def test_plot_returns_figure(self, normalize: bool) -> None: + """plot() must not crash on the integer matrix produced by from_tensors.""" + targets = [ + np.array( + [ + [0.0, 0.0, 3.0, 3.0, 0], + [6.0, 1.0, 8.0, 3.0, 1], + ], + dtype=np.float32, + ) + ] + predictions = [ + np.array( + [ + [0.0, 0.0, 3.0, 3.0, 0, 0.9], + ], + dtype=np.float32, + ) + ] + confusion_matrix = ConfusionMatrix.from_tensors( + predictions=predictions, + targets=targets, + classes=["person", "dog"], + ) + + fig = confusion_matrix.plot(normalize=normalize) + + assert fig is not None + plt.close(fig) diff --git a/tests/metrics/test_mean_average_precision.py b/tests/metrics/test_mean_average_precision.py index 6b072de2..20fb5b19 100644 --- a/tests/metrics/test_mean_average_precision.py +++ b/tests/metrics/test_mean_average_precision.py @@ -1,4 +1,5 @@ import numpy as np +import pytest from supervision.config import ORIENTED_BOX_COORDINATES from supervision.detection.core import Detections @@ -6,6 +7,30 @@ from supervision.metrics.core import MetricTarget from supervision.metrics.mean_average_precision import MeanAveragePrecision +def _mask_detections( + row_slice: slice, confidence: bool = False, mask_shape: tuple[int, int] = (32, 32) +) -> Detections: + """Build single-detection `Detections` with a mask filling the given rows.""" + mask = np.zeros((1, *mask_shape), dtype=bool) + mask[0, row_slice, :] = True + return Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]) if confidence else None, + mask=mask, + ) + + +def _obb_detections(corners: list[list[int]], confidence: bool = False) -> Detections: + """Build single-detection `Detections` with the given oriented box corners.""" + return Detections( + xyxy=np.array([[0, 0, 30, 30]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]) if confidence else None, + data={ORIENTED_BOX_COORDINATES: np.array([corners], dtype=np.float32)}, + ) + + class TestMeanAveragePrecision: def test_single_perfect_detection(self, detections_50_50, targets_50_50): """Test that single perfect detection gets 1.0 mAP (not 0.0 due to ID=0 bug)""" @@ -36,14 +61,7 @@ class TestMeanAveragePrecision: assert abs(result.map50_95 - 1.0) < 1e-6 def test_perfect_non_square_oriented_boxes_get_full_map(self): - """Smoke test: MeanAveragePrecision accepts non-square OBB inputs without error. - - NOTE: MeanAveragePrecision uses the COCO evaluator path - (box_iou_batch_with_jaccard) and does not route through - oriented_box_iou_batch regardless of metric_target. - This test verifies API acceptance and map50_95=1.0 via - xyxy COCO IoU, not OBB IoU. - """ + """Perfect non-square OBB predictions score full mAP via OBB IoU.""" obb = np.array( [[[10, 0], [0, 1], [30, 4], [40, 3]]], dtype=np.float32, @@ -333,3 +351,252 @@ class TestMeanAveragePrecision: assert result.small_objects.map50_95 == -1 assert result.medium_objects.map50_95 == -1 assert result.large_objects.map50_95 == -1 + + +class TestMeanAveragePrecisionMasks: + @pytest.mark.parametrize( + ("prediction_rows", "target_rows", "expected_map50"), + [ + pytest.param(slice(0, 16), slice(0, 16), 1.0, id="matching-masks"), + pytest.param(slice(0, 16), slice(16, 32), 0.0, id="disjoint-masks"), + ], + ) + def test_map50_follows_mask_overlap( + self, prediction_rows: slice, target_rows: slice, expected_map50: float + ) -> None: + """With MASKS target, map50 must reflect mask IoU, not identical boxes.""" + predictions = _mask_detections(prediction_rows, confidence=True) + targets = _mask_detections(target_rows) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(expected_map50, abs=1e-6) + + def test_missing_masks_raise(self) -> None: + """With MASKS target, detections without masks must raise ValueError.""" + predictions = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]), + ) + targets = _mask_detections(slice(0, 16)) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + metric.update([predictions], [targets]) + + with pytest.raises(ValueError, match="MASKS"): + metric.compute() + + def test_mask_pixel_count_drives_size_buckets(self) -> None: + """With MASKS target, object size buckets use mask area, not bbox area.""" + # bbox area is 100*100 = 10000 (large), mask area is 30*30 = 900 (small) + mask = np.zeros((1, 120, 120), dtype=bool) + mask[0, 10:40, 10:40] = True + predictions = Detections( + xyxy=np.array([[0, 0, 100, 100]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]), + mask=mask, + ) + targets = Detections( + xyxy=np.array([[0, 0, 100, 100]], dtype=np.float64), + class_id=np.array([0]), + mask=mask.copy(), + ) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + + result = metric.update([predictions], [targets]).compute() + + assert result.small_objects.map50 == pytest.approx(1.0, abs=1e-6) + assert result.large_objects.map50 == -1 + + def test_boxes_target_ignores_masks(self) -> None: + """With default BOXES target, disjoint masks must not affect the score.""" + predictions = _mask_detections(slice(0, 16), confidence=True) + targets = _mask_detections(slice(16, 32)) + metric = MeanAveragePrecision() + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(1.0, abs=1e-6) + + +class TestMeanAveragePrecisionOrientedBoundingBoxes: + @pytest.mark.parametrize( + ("prediction_corners", "target_corners", "expected_map50"), + [ + pytest.param( + [[0, 0], [10, 0], [10, 10], [0, 10]], + [[0, 0], [10, 0], [10, 10], [0, 10]], + 1.0, + id="matching-obb", + ), + pytest.param( + [[0, 0], [10, 0], [10, 10], [0, 10]], + [[20, 20], [30, 20], [30, 30], [20, 30]], + 0.0, + id="disjoint-obb", + ), + ], + ) + def test_map50_follows_oriented_box_overlap( + self, + prediction_corners: list[list[int]], + target_corners: list[list[int]], + expected_map50: float, + ) -> None: + """With OBB target, map50 must reflect OBB IoU, not identical boxes.""" + predictions = _obb_detections(prediction_corners, confidence=True) + targets = _obb_detections(target_corners) + metric = MeanAveragePrecision( + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES + ) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(expected_map50, abs=1e-6) + + def test_missing_oriented_boxes_raise(self) -> None: + """With OBB target, detections without OBB data must raise ValueError.""" + predictions = Detections( + xyxy=np.array([[0, 0, 30, 30]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]), + ) + targets = _obb_detections([[0, 0], [10, 0], [10, 10], [0, 10]]) + metric = MeanAveragePrecision( + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES + ) + metric.update([predictions], [targets]) + + with pytest.raises(ValueError, match=ORIENTED_BOX_COORDINATES): + metric.compute() + + def test_cross_matched_obb_orients_iou_correctly(self) -> None: + """2x2 cross-match: pred0->target1, pred1->target0 must both score as TP. + + A transposed (gt, dt) matrix would yield 0 IoU for every pair; map50=0. + Passing asserts the (dt, gt) orientation is correct end-to-end. + """ + box_tl = np.array([[0, 0], [10, 0], [10, 10], [0, 10]], dtype=np.float32) + box_br = np.array([[20, 20], [30, 20], [30, 30], [20, 30]], dtype=np.float32) + targets = Detections( + xyxy=np.array([[0, 0, 10, 10], [20, 20, 30, 30]], dtype=np.float64), + class_id=np.array([0, 0]), + data={ORIENTED_BOX_COORDINATES: np.stack([box_tl, box_br])}, + ) + # Predictions deliberately swapped: pred0 matches target1, pred1 matches target0 + predictions = Detections( + xyxy=np.array([[20, 20, 30, 30], [0, 0, 10, 10]], dtype=np.float64), + class_id=np.array([0, 0]), + confidence=np.array([0.9, 0.8]), + data={ORIENTED_BOX_COORDINATES: np.stack([box_br, box_tl])}, + ) + metric = MeanAveragePrecision( + metric_target=MetricTarget.ORIENTED_BOUNDING_BOXES + ) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(1.0, abs=1e-6) + + +class TestMeanAveragePrecisionMasksCrowdBranch: + """Tests for the crowd-aware Jaccard path in _mask_iou_with_jaccard.""" + + def test_crowd_gt_ignores_contained_detection(self) -> None: + """Detection inside a crowd GT is ignored (not FP) with Jaccard crowd IoU. + + Without Jaccard: small pred's standard IoU with crowd GT is 0.25 < 0.5, + so pred0 is a FP, which reduces map50. With Jaccard: IoU = 1.0, pred0 is + matched to crowd and ignored, so only pred1 (perfect TP) is scored -> map50=1.0. + """ + mask_normal = np.zeros((1, 32, 32), dtype=bool) + mask_normal[0, :16, :] = True # normal GT: top half + mask_crowd = np.ones((1, 32, 32), dtype=bool) # crowd GT: full image + + targets = Detections( + xyxy=np.array([[0, 0, 32, 16], [0, 0, 32, 32]], dtype=np.float64), + class_id=np.array([0, 0]), + mask=np.concatenate([mask_normal, mask_crowd]), + data={"iscrowd": np.array([0, 1], dtype=np.int64)}, + ) + # pred0 (conf=0.9): bottom quarter - inside crowd, no overlap with normal GT + mask_pred0 = np.zeros((1, 32, 32), dtype=bool) + mask_pred0[0, 16:24, :] = True + # pred1 (conf=0.8): exact match with normal GT + mask_pred1 = np.zeros((1, 32, 32), dtype=bool) + mask_pred1[0, :16, :] = True + + predictions = Detections( + xyxy=np.array([[0, 16, 32, 24], [0, 0, 32, 16]], dtype=np.float64), + class_id=np.array([0, 0]), + confidence=np.array([0.9, 0.8]), + mask=np.concatenate([mask_pred0, mask_pred1]), + ) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(1.0, abs=1e-6) + + def test_normal_gt_matched_correctly_alongside_crowd_gt(self) -> None: + """Normal GT is matched and scored when a crowd GT is also present.""" + mask_normal = np.zeros((1, 32, 32), dtype=bool) + mask_normal[0, :16, :] = True + mask_crowd = np.ones((1, 32, 32), dtype=bool) + + targets = Detections( + xyxy=np.array([[0, 0, 32, 16], [0, 0, 32, 32]], dtype=np.float64), + class_id=np.array([0, 0]), + mask=np.concatenate([mask_normal, mask_crowd]), + data={"iscrowd": np.array([0, 1], dtype=np.int64)}, + ) + mask_pred = np.zeros((1, 32, 32), dtype=bool) + mask_pred[0, :16, :] = True # exact match with normal GT + + predictions = Detections( + xyxy=np.array([[0, 0, 32, 16]], dtype=np.float64), + class_id=np.array([0]), + confidence=np.array([0.9]), + mask=mask_pred, + ) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(1.0, abs=1e-6) + + +class TestMeanAveragePrecisionMasksOrientation: + """Tests that the (dt, gt) IoU-matrix orientation is correct end-to-end.""" + + def test_cross_matched_masks_orient_iou_correctly(self) -> None: + """2x2 cross-match: pred0->target1, pred1->target0 must both score as TP. + + A transposed (gt, dt) matrix would yield 0 IoU for every pair; map50=0. + Passing asserts the (dt, gt) orientation is correct end-to-end. + """ + top_mask = np.zeros((1, 32, 32), dtype=bool) + top_mask[0, :16, :] = True + bottom_mask = np.zeros((1, 32, 32), dtype=bool) + bottom_mask[0, 16:, :] = True + + # target0 = top half, target1 = bottom half + targets = Detections( + xyxy=np.array([[0, 0, 32, 16], [0, 16, 32, 32]], dtype=np.float64), + class_id=np.array([0, 0]), + mask=np.concatenate([top_mask, bottom_mask]), + ) + # Predictions deliberately swapped: pred0=bottom, pred1=top + predictions = Detections( + xyxy=np.array([[0, 16, 32, 32], [0, 0, 32, 16]], dtype=np.float64), + class_id=np.array([0, 0]), + confidence=np.array([0.9, 0.8]), + mask=np.concatenate([bottom_mask, top_mask]), + ) + metric = MeanAveragePrecision(metric_target=MetricTarget.MASKS) + + result = metric.update([predictions], [targets]).compute() + + assert result.map50 == pytest.approx(1.0, abs=1e-6) diff --git a/tests/metrics/test_oriented_bounding_box_metrics.py b/tests/metrics/test_oriented_bounding_box_metrics.py index 9b7a96db..54ac43e9 100644 --- a/tests/metrics/test_oriented_bounding_box_metrics.py +++ b/tests/metrics/test_oriented_bounding_box_metrics.py @@ -48,12 +48,8 @@ def test_perfect_non_square_oriented_boxes_score_as_perfect( def test_mean_average_precision_accepts_obb_metric_target() -> None: - """Smoke test: MeanAveragePrecision accepts metric_target=ORIENTED_BOUNDING_BOXES. - - NOTE: MeanAveragePrecision uses the COCO evaluator path (box_iou_batch_with_jaccard) - and does not route through oriented_box_iou_batch regardless of metric_target. - This test verifies API acceptance only, not OBB IoU correctness. - """ + """MeanAveragePrecision routes metric_target=ORIENTED_BOUNDING_BOXES through + oriented_box_iou_batch; perfect OBB predictions score 1.0.""" predictions = _non_square_obb_detections(confidence=True) targets = _non_square_obb_detections()