From 072f78471c12d1e12f35cece98e99cb2d4dd2e32 Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Wed, 8 Jul 2026 08:56:34 +0200 Subject: [PATCH] fix: resolve medium dataset findings (#2408) - Reinstated NumPy-safe `Classifications` equality and ordered class-list comparisons in dataset equality. - Restored greedy matching plus size-bucket scoring for Precision, Recall, F1, and MeanAverageRecall, with regression coverage for the medium-object boundary case. - Filter size-bucket precision, recall, and F1 against target boxes so predictions no longer claim the bucket. - Preserve confidence order for bucketed mAR@K scoring and return zero when a bucket has no support. - Add regression coverage for bucket matching, empty-support mAR, top-K limits, and missing-mask errors. --------- Co-authored-by: Codex --- docs/changelog.md | 2 + src/supervision/classification/core.py | 12 ++ src/supervision/dataset/core.py | 4 +- src/supervision/metrics/f1_score.py | 149 +++++++++++---- .../metrics/mean_average_recall.py | 110 +++++------ src/supervision/metrics/precision.py | 148 +++++++++++---- src/supervision/metrics/recall.py | 140 ++++++++++---- src/supervision/metrics/utils/matching.py | 23 +++ tests/classification/test_core.py | 19 ++ tests/dataset/test_core.py | 50 ++++- tests/metrics/test_f1_score.py | 17 ++ tests/metrics/test_mean_average_recall.py | 27 +++ tests/metrics/test_precision.py | 17 ++ tests/metrics/test_recall.py | 17 ++ tests/metrics/test_size_bucket_regressions.py | 177 ++++++++++++++++++ 15 files changed, 740 insertions(+), 172 deletions(-) create mode 100644 tests/metrics/test_size_bucket_regressions.py diff --git a/docs/changelog.md b/docs/changelog.md index 83130d28..532c55e4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -21,6 +21,7 @@ date_modified: 2026-07-06 - `sv.Classifications.from_timm` now softmaxes model logits before exposing confidence scores, matching `sv.Classifications.from_clip` and keeping timm confidences on a normalized probability scale. Thresholds calibrated against raw logits may need retuning. - `sv.download_assets` now verifies MD5 hashes after fresh downloads and retries once when the downloaded payload is corrupted instead of accepting a bad file. - Fixed metrics scoring edge cases: legacy `sv.MeanAveragePrecision` now uses COCO 101-point AP averaging, `sv.ConfusionMatrix` rejects invalid class ids instead of wrapping them through `int16`/negative indexing, `sv.MeanAveragePrecision` preserves user-provided target `ignore` flags, and `sv.MeanAverageRecallResult.recall_per_class` now exposes per-class recall for each max-detection cutoff. +- Fixed [#2408](https://github.com/roboflow/supervision/pull/2408): `sv.Precision`, `sv.Recall`, `sv.F1Score`, and `sv.MeanAverageRecall` now score size buckets by filtering targets only while leaving predictions eligible to match bucket targets. This preserves bucket matches that would otherwise be stolen by out-of-bucket filtering and keeps mAR top-K ranking intact. - `sv.ByteTrack` no longer mutates input `Detections` while assigning tracker IDs. It now keeps detections at the activation-threshold boundary eligible for matching, avoids impossible new-track thresholds above score `1.0`, ignores invalid zero-area/non-finite tensor boxes before Kalman updates, and does not emit unconfirmed `-1` IDs from first-frame tensor updates. - Fixed [#2402](https://github.com/roboflow/supervision/pull/2402): `sv.KeyPoints.as_detections` now accepts NumPy arrays, tuples, and generators in `selected_keypoint_indices` without ambiguous truth-value errors; empty index iterables select all keypoints. Valid zero-area skeletons are preserved, while all-zero and non-finite-only skeletons are filtered out. - Fixed [#2407](https://github.com/roboflow/supervision/pull/2407): `sv.ColorPalette.by_idx()` now raises a clear `ValueError` when called on an empty palette instead of leaking a `ZeroDivisionError`. Non-empty palettes keep the existing index-wrapping behavior. @@ -39,6 +40,7 @@ date_modified: 2026-07-06 ### Changed - Performance [#2383](https://github.com/roboflow/supervision/pull/2383): `sv.Detections.merge()` on mixed dense `ndarray` + `CompactMask` inputs now returns a `CompactMask` instead of a dense `ndarray`. Previously (0.29.0/0.29.1) the mixed path fell back to `np.vstack`, allocating a full `(N, H, W)` array; the new path converts dense inputs to `CompactMask` without materialising the full stack (~2 500× less peak memory, ~13× faster on 1080p / 40 detections). **Behavior change**: code that checks `isinstance(merged.mask, np.ndarray)` or calls bare ndarray methods (`.astype`, `.reshape`, `.ravel`) on a mixed-merge result will need to be updated. The all-dense path is unchanged and still returns `ndarray`. +- `DetectionDataset` and `ClassificationDataset` equality now compare the ordered `classes` lists directly instead of treating class labels as an unordered set. This keeps equality aligned with `class_id` indexing semantics, where class position is part of the dataset contract. ### 0.29.1 Jun 23, 2026 diff --git a/src/supervision/classification/core.py b/src/supervision/classification/core.py index 00789274..9b7a15b4 100644 --- a/src/supervision/classification/core.py +++ b/src/supervision/classification/core.py @@ -43,6 +43,18 @@ class Classifications: _validate_class_ids(self.class_id, n) _validate_confidence(self.confidence, n) + def __eq__(self, other: object) -> bool: + """ + Compare classifications by value across numpy-backed fields. + """ + if not isinstance(other, Classifications): + return NotImplemented + if not np.array_equal(self.class_id, other.class_id): + return False + if self.confidence is None or other.confidence is None: + return self.confidence is other.confidence + return bool(np.array_equal(self.confidence, other.confidence)) + def __len__(self) -> int: """ Returns the number of classifications. diff --git a/src/supervision/dataset/core.py b/src/supervision/dataset/core.py index 95458447..9ec005c0 100644 --- a/src/supervision/dataset/core.py +++ b/src/supervision/dataset/core.py @@ -184,7 +184,7 @@ class DetectionDataset(BaseDataset): if not isinstance(other, DetectionDataset): return False - if set(self.classes) != set(other.classes): + if self.classes != other.classes: return False if self.image_paths != other.image_paths: @@ -1105,7 +1105,7 @@ class ClassificationDataset(BaseDataset): if not isinstance(other, ClassificationDataset): return False - if set(self.classes) != set(other.classes): + if self.classes != other.classes: return False if self.image_paths != other.image_paths: diff --git a/src/supervision/metrics/f1_score.py b/src/supervision/metrics/f1_score.py index 0f8b1e42..f356a146 100644 --- a/src/supervision/metrics/f1_score.py +++ b/src/supervision/metrics/f1_score.py @@ -18,7 +18,9 @@ from supervision.detection.utils.iou_and_nms import ( ) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import AveragingMethod, Metric, MetricTarget -from supervision.metrics.utils.matching import _greedy_match +from supervision.metrics.utils.matching import ( + _match_detection_batch_with_target_indices, +) from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -131,43 +133,61 @@ class F1Score(Metric["F1ScoreResult"]): The F1 score metric result. """ result = self._compute(self._predictions_list, self._targets_list) - - small_predictions, small_targets = self._filter_predictions_and_targets_by_size( + result.small_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL ) - result.small_objects = self._compute(small_predictions, small_targets) - - medium_predictions, medium_targets = ( - self._filter_predictions_and_targets_by_size( - self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM - ) + result.medium_objects = self._compute( + self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM ) - result.medium_objects = self._compute(medium_predictions, medium_targets) - - large_predictions, large_targets = self._filter_predictions_and_targets_by_size( + result.large_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE ) - result.large_objects = self._compute(large_predictions, large_targets) return result def _compute( - self, predictions_list: list[Detections], targets_list: list[Detections] + self, + predictions_list: list[Detections], + targets_list: list[Detections], + size_category: ObjectSizeCategory = ObjectSizeCategory.ANY, ) -> F1ScoreResult: """Build per-image stats tuples and delegate to class-level computation. - Each stats tuple is ``(matches, confidence, class_ids, true_class_ids)``: + Each stats tuple is + ``(matches, ignored_matches, confidence, class_ids, true_class_ids)``: - Both empty: skip (no information). - Targets empty, predictions present: all predictions are FPs; true_class_ids is ``zeros((0,))``. - Targets present: IoU matching produces ``matches`` array. """ + if size_category != ObjectSizeCategory.ANY: + # Score the requested bucket on bucket-filtered targets so detections + # outside the bucket cannot consume the only available target. + targets_list = [ + self._filter_detections_by_size(targets, size_category) + for targets in targets_list + ] + size_category = ObjectSizeCategory.ANY + iou_thresholds = np.linspace(0.5, 0.95, 10, dtype=np.float32) stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) target_contents = self._detections_content(targets) + prediction_size_mask = np.ones(len(predictions), dtype=bool) + target_size_mask = np.ones(len(targets), dtype=bool) + if size_category != ObjectSizeCategory.ANY: + if len(predictions) > 0: + prediction_size_mask = ( + get_detection_size_category(predictions, self._metric_target) + == size_category.value + ) + if len(targets) > 0: + target_size_mask = ( + get_detection_size_category(targets, self._metric_target) + == size_category.value + ) if len(targets) == 0 and len(predictions) > 0: # Only predictions are present (e.g. a background image); every @@ -177,14 +197,23 @@ class F1Score(Metric["F1ScoreResult"]): "F1Score metric requires `class_id` and `confidence` " "on predictions." ) - prediction_class_ids = np.asarray(predictions.class_id, dtype=np.int32) + prediction_class_ids = np.asarray(predictions.class_id, dtype=np.int32)[ + prediction_size_mask + ] prediction_confidence = np.asarray( predictions.confidence, dtype=np.float32 - ) + )[prediction_size_mask] + if len(prediction_class_ids) == 0: + continue stats.append( ( np.zeros( - (len(predictions), iou_thresholds.size), dtype=np.bool_ + (len(prediction_class_ids), iou_thresholds.size), + dtype=np.bool_, + ), + np.zeros( + (len(prediction_class_ids), iou_thresholds.size), + dtype=np.bool_, ), prediction_confidence, prediction_class_ids, @@ -198,12 +227,18 @@ class F1Score(Metric["F1ScoreResult"]): "and targets." ) if len(predictions) == 0: + target_class_ids = np.asarray(targets.class_id, dtype=np.int32)[ + target_size_mask + ] + if len(target_class_ids) == 0: + continue stats.append( ( + np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0,), dtype=np.float32), np.zeros((0,), dtype=int), - targets.class_id, + target_class_ids, ) ) @@ -238,15 +273,48 @@ class F1Score(Metric["F1ScoreResult"]): "Unsupported metric target for IoU calculation" ) - matches = self._match_detection_batch( - prediction_class_ids, - target_class_ids, - iou, - iou_thresholds, + matches, matched_target_indices = ( + _match_detection_batch_with_target_indices( + prediction_class_ids, + target_class_ids, + iou, + iou_thresholds, + ) ) + ignored_matches = np.zeros_like(matches, dtype=bool) + if size_category != ObjectSizeCategory.ANY: + valid_target_match = matched_target_indices >= 0 + matched_scored_target = np.zeros_like(matches, dtype=bool) + if np.any(valid_target_match): + matched_scored_target[valid_target_match] = ( + target_size_mask[ + matched_target_indices[valid_target_match] + ] + ) + prediction_scored = ( + prediction_size_mask[:, None] | matched_scored_target + ) + ignored_matches = ~prediction_scored | ( + valid_target_match & ~matched_scored_target + ) + prediction_keep = np.any(~ignored_matches, axis=1) + matches = ( + matches[prediction_keep] + & matched_scored_target[prediction_keep] + ) + ignored_matches = ignored_matches[prediction_keep] + prediction_confidence = prediction_confidence[prediction_keep] + prediction_class_ids = prediction_class_ids[prediction_keep] + target_class_ids = target_class_ids[target_size_mask] + if ( + len(prediction_class_ids) == 0 + and len(target_class_ids) == 0 + ): + continue stats.append( ( matches, + ignored_matches, prediction_confidence, prediction_class_ids, target_class_ids, @@ -286,6 +354,7 @@ class F1Score(Metric["F1ScoreResult"]): def _compute_f1_for_classes( self, matches: npt.NDArray[np.bool_], + ignored_matches: npt.NDArray[np.bool_], prediction_confidence: npt.NDArray[np.float32], prediction_class_ids: npt.NDArray[np.int32], true_class_ids: npt.NDArray[np.int32], @@ -301,6 +370,7 @@ class F1Score(Metric["F1ScoreResult"]): """ sorted_indices = np.argsort(-prediction_confidence) matches = matches[sorted_indices] + ignored_matches = ignored_matches[sorted_indices] prediction_class_ids = prediction_class_ids[sorted_indices] # Predictions whose class never appears in the ground truth are still # false positives, so include those classes in the confusion matrix @@ -314,7 +384,7 @@ class F1Score(Metric["F1ScoreResult"]): # Shape: PxTh,P,C,C -> CxThx3 confusion_matrix = self._compute_confusion_matrix( - matches, prediction_class_ids, unique_classes, class_counts + matches, ignored_matches, prediction_class_ids, unique_classes, class_counts ) # Shape: CxThx3 -> CxTh @@ -345,25 +415,15 @@ class F1Score(Metric["F1ScoreResult"]): iou: npt.NDArray[np.float32], iou_thresholds: npt.NDArray[np.float32], ) -> npt.NDArray[np.bool_]: - num_predictions, num_iou_levels = ( - predictions_classes.shape[0], - iou_thresholds.shape[0], + result_correct, _ = _match_detection_batch_with_target_indices( + predictions_classes, target_classes, iou, iou_thresholds ) - correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) - correct_class = target_classes[:, None] == predictions_classes - - for i, iou_level in enumerate(iou_thresholds): - matched_indices = np.where((iou >= iou_level) & correct_class) - - for t, p in _greedy_match(iou, matched_indices): - correct[p, i] = True - - result_correct: npt.NDArray[np.bool_] = correct return result_correct @staticmethod def _compute_confusion_matrix( sorted_matches: npt.NDArray[np.bool_], + sorted_ignored_matches: npt.NDArray[np.bool_], sorted_prediction_class_ids: npt.NDArray[np.int32], unique_classes: npt.NDArray[np.int32], class_counts: npt.NDArray[np.int32], @@ -377,6 +437,8 @@ class F1Score(Metric["F1ScoreResult"]): Args: sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. + sorted_ignored_matches: shape (P, Th), that is True + if the prediction should not affect the given IoU threshold. sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. unique_classes: shape (C,), containing the unique @@ -406,11 +468,13 @@ class F1Score(Metric["F1ScoreResult"]): false_negatives = np.full(num_thresholds, num_true) elif num_true == 0: true_positives = np.zeros(num_thresholds) - false_positives = np.full(num_thresholds, num_predictions) + false_positives = (~sorted_ignored_matches[is_class]).sum(0) false_negatives = np.zeros(num_thresholds) else: true_positives = sorted_matches[is_class].sum(0) - false_positives = (1 - sorted_matches[is_class]).sum(0) + false_positives = ( + ~sorted_matches[is_class] & ~sorted_ignored_matches[is_class] + ).sum(0) false_negatives = num_true - true_positives confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 @@ -464,6 +528,11 @@ class F1Score(Metric["F1ScoreResult"]): if detections.mask is not None: # detections.mask is NDArray[bool] | CompactMask; return as-is. return detections.mask + if len(detections) > 0: + raise ValueError( + "F1Score with `MetricTarget.MASKS` requires detections to " + "include masks." + ) return self._make_empty_content() if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) diff --git a/src/supervision/metrics/mean_average_recall.py b/src/supervision/metrics/mean_average_recall.py index c7caa201..810c071b 100644 --- a/src/supervision/metrics/mean_average_recall.py +++ b/src/supervision/metrics/mean_average_recall.py @@ -18,7 +18,9 @@ from supervision.detection.utils.iou_and_nms import ( ) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import Metric, MetricTarget -from supervision.metrics.utils.matching import _greedy_match +from supervision.metrics.utils.matching import ( + _match_detection_batch_with_target_indices, +) from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -368,36 +370,39 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): The Mean Average Recall metric result. """ result = self._compute(self._predictions_list, self._targets_list) - - small_predictions, small_targets = self._filter_predictions_and_targets_by_size( + result.small_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL ) - result.small_objects = self._compute(small_predictions, small_targets) - - medium_predictions, medium_targets = ( - self._filter_predictions_and_targets_by_size( - self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM - ) + result.medium_objects = self._compute( + self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM ) - result.medium_objects = self._compute(medium_predictions, medium_targets) - - large_predictions, large_targets = self._filter_predictions_and_targets_by_size( + result.large_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE ) - result.large_objects = self._compute(large_predictions, large_targets) return result def _compute( - self, predictions_list: list[Detections], targets_list: list[Detections] + self, + predictions_list: list[Detections], + targets_list: list[Detections], + size_category: ObjectSizeCategory = ObjectSizeCategory.ANY, ) -> MeanAverageRecallResult: + if size_category != ObjectSizeCategory.ANY: + # Score the requested bucket on bucket-filtered targets so detections + # outside the bucket cannot consume the only available target. + targets_list = [ + self._filter_detections_by_size(targets, size_category) + for targets in targets_list + ] + size_category = ObjectSizeCategory.ANY + iou_thresholds = np.linspace(0.5, 0.95, 10, dtype=np.float32) stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) target_contents = self._detections_content(targets) - if len(targets) > 0: if predictions.class_id is None or targets.class_id is None: raise ValueError( @@ -405,12 +410,16 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): "predictions and targets." ) if len(predictions) == 0: + target_class_ids = np.asarray(targets.class_id, dtype=np.int32) + if len(target_class_ids) == 0: + continue stats.append( ( + np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0,), dtype=int), np.zeros((0,), dtype=int), - targets.class_id, + target_class_ids, ) ) @@ -446,18 +455,20 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): "Unsupported metric target for IoU calculation" ) - matches = self._match_detection_batch( + matches, _ = _match_detection_batch_with_target_indices( prediction_class_ids, target_class_ids, iou, iou_thresholds, ) + ignored_matches = np.zeros_like(matches, dtype=bool) sorted_indices = np.argsort(-prediction_confidence) stats.append( ( matches[sorted_indices], - np.arange(len(predictions)), + ignored_matches[sorted_indices], + np.arange(len(prediction_confidence)), prediction_class_ids[sorted_indices], target_class_ids, ) @@ -499,6 +510,7 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): def _compute_average_recall_for_classes( self, matches: npt.NDArray[np.bool_], + ignored_matches: npt.NDArray[np.bool_], prediction_indices: npt.NDArray[np.int32], prediction_class_ids: npt.NDArray[np.int32], true_class_ids: npt.NDArray[np.int32], @@ -509,12 +521,23 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): ]: unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) + if unique_classes.size == 0: + max_detection_count = self.max_detections.shape[0] + num_thresholds = matches.shape[1] + return ( + np.zeros(max_detection_count, dtype=np.float64), + np.zeros((max_detection_count, 0, num_thresholds), dtype=np.float64), + unique_classes, + ) + recalls_at_k: list[npt.NDArray[np.float64]] = [] for max_detections in self.max_detections: # Shape: PxTh,P,C,C -> CxThx3 + is_within_limit = prediction_indices < max_detections confusion_matrix = self._compute_confusion_matrix( - matches[prediction_indices < max_detections], - prediction_class_ids[prediction_indices < max_detections], + matches[is_within_limit], + ignored_matches[is_within_limit], + prediction_class_ids[is_within_limit], unique_classes, class_counts, ) @@ -539,24 +562,15 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): iou: npt.NDArray[np.float32], iou_thresholds: npt.NDArray[np.float32], ) -> npt.NDArray[np.bool_]: - num_predictions, num_iou_levels = ( - predictions_classes.shape[0], - iou_thresholds.shape[0], + result_correct, _ = _match_detection_batch_with_target_indices( + predictions_classes, target_classes, iou, iou_thresholds ) - correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) - correct_class = target_classes[:, None] == predictions_classes - - for i, iou_level in enumerate(iou_thresholds): - matched_indices = np.where((iou >= iou_level) & correct_class) - - for t, p in _greedy_match(iou, matched_indices): - correct[p, i] = True - result_correct: npt.NDArray[np.bool_] = correct return result_correct @staticmethod def _compute_confusion_matrix( sorted_matches: npt.NDArray[np.bool_], + sorted_ignored_matches: npt.NDArray[np.bool_], sorted_prediction_class_ids: npt.NDArray[np.int32], unique_classes: npt.NDArray[np.integer], class_counts: npt.NDArray[np.integer], @@ -570,6 +584,8 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): Args: sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. + sorted_ignored_matches: shape (P, Th), that is True + if the prediction should not affect the given IoU threshold. sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. unique_classes: shape (C,), containing the unique @@ -601,13 +617,14 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): false_negatives = np.full(num_thresholds, num_true) elif num_true == 0: true_positives = np.zeros(num_thresholds) - false_positives = np.full(num_thresholds, num_predictions) + false_positives = (~sorted_ignored_matches[is_class]).sum(0) false_negatives = np.zeros(num_thresholds) else: limited_matches = sorted_matches[is_class] + limited_ignored_matches = sorted_ignored_matches[is_class] true_positives = limited_matches.sum(0) - false_positives = (1 - limited_matches).sum(0) + false_positives = (~limited_matches & ~limited_ignored_matches).sum(0) false_negatives = num_true - true_positives confusion_matrix[class_idx] = np.stack( @@ -660,6 +677,11 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): if detections.mask is not None: # detections.mask is NDArray[bool] | CompactMask; return as-is. return detections.mask + if len(detections) > 0: + raise ValueError( + "MeanAverageRecall with `MetricTarget.MASKS` requires " + "detections to include masks." + ) return self._make_empty_content() if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) @@ -709,23 +731,3 @@ class MeanAverageRecall(Metric["MeanAverageRecallResult"]): new_detections.data[key] = np.array(value)[size_mask] return new_detections - - def _filter_predictions_and_targets_by_size( - self, - predictions_list: list[Detections], - targets_list: list[Detections], - size_category: ObjectSizeCategory, - ) -> tuple[list[Detections], list[Detections]]: - """ - Filter predictions and targets by object size category. - """ - new_predictions_list = [] - new_targets_list = [] - for predictions, targets in zip(predictions_list, targets_list): - new_predictions_list.append( - self._filter_detections_by_size(predictions, size_category) - ) - new_targets_list.append( - self._filter_detections_by_size(targets, size_category) - ) - return new_predictions_list, new_targets_list diff --git a/src/supervision/metrics/precision.py b/src/supervision/metrics/precision.py index 978f60fc..f2101d20 100644 --- a/src/supervision/metrics/precision.py +++ b/src/supervision/metrics/precision.py @@ -17,7 +17,9 @@ from supervision.detection.utils.iou_and_nms import ( ) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import AveragingMethod, Metric, MetricTarget -from supervision.metrics.utils.matching import _greedy_match +from supervision.metrics.utils.matching import ( + _match_detection_batch_with_target_indices, +) from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -133,43 +135,61 @@ class Precision(Metric["PrecisionResult"]): The precision metric result. """ result = self._compute(self._predictions_list, self._targets_list) - - small_predictions, small_targets = self._filter_predictions_and_targets_by_size( + result.small_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL ) - result.small_objects = self._compute(small_predictions, small_targets) - - medium_predictions, medium_targets = ( - self._filter_predictions_and_targets_by_size( - self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM - ) + result.medium_objects = self._compute( + self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM ) - result.medium_objects = self._compute(medium_predictions, medium_targets) - - large_predictions, large_targets = self._filter_predictions_and_targets_by_size( + result.large_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE ) - result.large_objects = self._compute(large_predictions, large_targets) return result def _compute( - self, predictions_list: list[Detections], targets_list: list[Detections] + self, + predictions_list: list[Detections], + targets_list: list[Detections], + size_category: ObjectSizeCategory = ObjectSizeCategory.ANY, ) -> PrecisionResult: """Build per-image stats tuples and delegate to class-level computation. - Each stats tuple is ``(matches, confidence, class_ids, true_class_ids)``: + Each stats tuple is + ``(matches, ignored_matches, confidence, class_ids, true_class_ids)``: - Both empty: skip (no information). - Targets empty, predictions present: all predictions are FPs; true_class_ids is ``zeros((0,))``. - Targets present: IoU matching produces ``matches`` array. """ + if size_category != ObjectSizeCategory.ANY: + # Score the requested bucket on bucket-filtered targets so detections + # outside the bucket cannot consume the only available target. + targets_list = [ + self._filter_detections_by_size(targets, size_category) + for targets in targets_list + ] + size_category = ObjectSizeCategory.ANY + iou_thresholds = np.linspace(0.5, 0.95, 10, dtype=np.float32) stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) target_contents = self._detections_content(targets) + prediction_size_mask = np.ones(len(predictions), dtype=bool) + target_size_mask = np.ones(len(targets), dtype=bool) + if size_category != ObjectSizeCategory.ANY: + if len(predictions) > 0: + prediction_size_mask = ( + get_detection_size_category(predictions, self._metric_target) + == size_category.value + ) + if len(targets) > 0: + target_size_mask = ( + get_detection_size_category(targets, self._metric_target) + == size_category.value + ) if len(targets) == 0 and len(predictions) > 0: # Only predictions are present (e.g. a background image); every @@ -179,14 +199,23 @@ class Precision(Metric["PrecisionResult"]): "Precision metric requires `class_id` and `confidence` " "on predictions." ) - prediction_class_ids = np.asarray(predictions.class_id, dtype=np.int32) + prediction_class_ids = np.asarray(predictions.class_id, dtype=np.int32)[ + prediction_size_mask + ] prediction_confidence = np.asarray( predictions.confidence, dtype=np.float32 - ) + )[prediction_size_mask] + if len(prediction_class_ids) == 0: + continue stats.append( ( np.zeros( - (len(predictions), iou_thresholds.size), dtype=np.bool_ + (len(prediction_class_ids), iou_thresholds.size), + dtype=np.bool_, + ), + np.zeros( + (len(prediction_class_ids), iou_thresholds.size), + dtype=np.bool_, ), prediction_confidence, prediction_class_ids, @@ -200,12 +229,18 @@ class Precision(Metric["PrecisionResult"]): "and targets." ) if len(predictions) == 0: + target_class_ids = np.asarray(targets.class_id, dtype=np.int32)[ + target_size_mask + ] + if len(target_class_ids) == 0: + continue stats.append( ( + np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0,), dtype=np.float32), np.zeros((0,), dtype=int), - targets.class_id, + target_class_ids, ) ) @@ -234,15 +269,48 @@ class Precision(Metric["PrecisionResult"]): "Unsupported metric target for IoU calculation" ) - matches = self._match_detection_batch( - prediction_class_ids, - target_class_ids, - iou, - iou_thresholds, + matches, matched_target_indices = ( + _match_detection_batch_with_target_indices( + prediction_class_ids, + target_class_ids, + iou, + iou_thresholds, + ) ) + ignored_matches = np.zeros_like(matches, dtype=bool) + if size_category != ObjectSizeCategory.ANY: + valid_target_match = matched_target_indices >= 0 + matched_scored_target = np.zeros_like(matches, dtype=bool) + if np.any(valid_target_match): + matched_scored_target[valid_target_match] = ( + target_size_mask[ + matched_target_indices[valid_target_match] + ] + ) + prediction_scored = ( + prediction_size_mask[:, None] | matched_scored_target + ) + ignored_matches = ~prediction_scored | ( + valid_target_match & ~matched_scored_target + ) + prediction_keep = np.any(~ignored_matches, axis=1) + matches = ( + matches[prediction_keep] + & matched_scored_target[prediction_keep] + ) + ignored_matches = ignored_matches[prediction_keep] + prediction_confidence = prediction_confidence[prediction_keep] + prediction_class_ids = prediction_class_ids[prediction_keep] + target_class_ids = target_class_ids[target_size_mask] + if ( + len(prediction_class_ids) == 0 + and len(target_class_ids) == 0 + ): + continue stats.append( ( matches, + ignored_matches, prediction_confidence, prediction_class_ids, target_class_ids, @@ -282,6 +350,7 @@ class Precision(Metric["PrecisionResult"]): def _compute_precision_for_classes( self, matches: npt.NDArray[np.bool_], + ignored_matches: npt.NDArray[np.bool_], prediction_confidence: npt.NDArray[np.float32], prediction_class_ids: npt.NDArray[np.int32], true_class_ids: npt.NDArray[np.int32], @@ -297,6 +366,7 @@ class Precision(Metric["PrecisionResult"]): """ sorted_indices = np.argsort(-prediction_confidence) matches = matches[sorted_indices] + ignored_matches = ignored_matches[sorted_indices] prediction_class_ids = prediction_class_ids[sorted_indices] # Predictions whose class never appears in the ground truth are still # false positives, so include those classes in the confusion matrix @@ -310,7 +380,7 @@ class Precision(Metric["PrecisionResult"]): # Shape: PxTh,P,C,C -> CxThx3 confusion_matrix = self._compute_confusion_matrix( - matches, prediction_class_ids, unique_classes, class_counts + matches, ignored_matches, prediction_class_ids, unique_classes, class_counts ) # Shape: CxThx3 -> CxTh @@ -343,24 +413,15 @@ class Precision(Metric["PrecisionResult"]): iou: npt.NDArray[np.float32], iou_thresholds: npt.NDArray[np.float32], ) -> npt.NDArray[np.bool_]: - num_predictions, num_iou_levels = ( - predictions_classes.shape[0], - iou_thresholds.shape[0], + result_correct, _ = _match_detection_batch_with_target_indices( + predictions_classes, target_classes, iou, iou_thresholds ) - correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) - correct_class = target_classes[:, None] == predictions_classes - - for i, iou_level in enumerate(iou_thresholds): - matched_indices = np.where((iou >= iou_level) & correct_class) - - for t, p in _greedy_match(iou, matched_indices): - correct[p, i] = True - result_correct: npt.NDArray[np.bool_] = correct return result_correct @staticmethod def _compute_confusion_matrix( sorted_matches: npt.NDArray[np.bool_], + sorted_ignored_matches: npt.NDArray[np.bool_], sorted_prediction_class_ids: npt.NDArray[np.int32], unique_classes: npt.NDArray[np.int32], class_counts: npt.NDArray[np.int32], @@ -374,6 +435,8 @@ class Precision(Metric["PrecisionResult"]): Args: sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. + sorted_ignored_matches: shape (P, Th), that is True + if the prediction should not affect the given IoU threshold. sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. unique_classes: shape (C,), containing the unique @@ -403,11 +466,13 @@ class Precision(Metric["PrecisionResult"]): false_negatives = np.full(num_thresholds, num_true) elif num_true == 0: true_positives = np.zeros(num_thresholds) - false_positives = np.full(num_thresholds, num_predictions) + false_positives = (~sorted_ignored_matches[is_class]).sum(0) false_negatives = np.zeros(num_thresholds) else: true_positives = sorted_matches[is_class].sum(0) - false_positives = (1 - sorted_matches[is_class]).sum(0) + false_positives = ( + ~sorted_matches[is_class] & ~sorted_ignored_matches[is_class] + ).sum(0) false_negatives = num_true - true_positives confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 @@ -455,6 +520,11 @@ class Precision(Metric["PrecisionResult"]): if self._metric_target == MetricTarget.MASKS: if detections.mask is not None: return cast(npt.NDArray[Any], detections.mask) + if len(detections) > 0: + raise ValueError( + "Precision with `MetricTarget.MASKS` requires detections to " + "include masks." + ) return self._make_empty_content() if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) diff --git a/src/supervision/metrics/recall.py b/src/supervision/metrics/recall.py index bb3e3ed1..f96aeb69 100644 --- a/src/supervision/metrics/recall.py +++ b/src/supervision/metrics/recall.py @@ -18,7 +18,9 @@ from supervision.detection.utils.iou_and_nms import ( ) from supervision.draw.color import LEGACY_COLOR_PALETTE from supervision.metrics.core import AveragingMethod, Metric, MetricTarget -from supervision.metrics.utils.matching import _greedy_match +from supervision.metrics.utils.matching import ( + _match_detection_batch_with_target_indices, +) from supervision.metrics.utils.object_size import ( ObjectSizeCategory, get_detection_size_category, @@ -134,35 +136,52 @@ class Recall(Metric["RecallResult"]): The recall metric result. """ result = self._compute(self._predictions_list, self._targets_list) - - small_predictions, small_targets = self._filter_predictions_and_targets_by_size( + result.small_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.SMALL ) - result.small_objects = self._compute(small_predictions, small_targets) - - medium_predictions, medium_targets = ( - self._filter_predictions_and_targets_by_size( - self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM - ) + result.medium_objects = self._compute( + self._predictions_list, self._targets_list, ObjectSizeCategory.MEDIUM ) - result.medium_objects = self._compute(medium_predictions, medium_targets) - - large_predictions, large_targets = self._filter_predictions_and_targets_by_size( + result.large_objects = self._compute( self._predictions_list, self._targets_list, ObjectSizeCategory.LARGE ) - result.large_objects = self._compute(large_predictions, large_targets) return result def _compute( - self, predictions_list: list[Detections], targets_list: list[Detections] + self, + predictions_list: list[Detections], + targets_list: list[Detections], + size_category: ObjectSizeCategory = ObjectSizeCategory.ANY, ) -> RecallResult: + if size_category != ObjectSizeCategory.ANY: + # Score the requested bucket on bucket-filtered targets so detections + # outside the bucket cannot consume the only available target. + targets_list = [ + self._filter_detections_by_size(targets, size_category) + for targets in targets_list + ] + size_category = ObjectSizeCategory.ANY + iou_thresholds = np.linspace(0.5, 0.95, 10, dtype=np.float32) stats: list[Any] = [] for predictions, targets in zip(predictions_list, targets_list): prediction_contents = self._detections_content(predictions) target_contents = self._detections_content(targets) + prediction_size_mask = np.ones(len(predictions), dtype=bool) + target_size_mask = np.ones(len(targets), dtype=bool) + if size_category != ObjectSizeCategory.ANY: + if len(predictions) > 0: + prediction_size_mask = ( + get_detection_size_category(predictions, self._metric_target) + == size_category.value + ) + if len(targets) > 0: + target_size_mask = ( + get_detection_size_category(targets, self._metric_target) + == size_category.value + ) if len(targets) > 0: if predictions.class_id is None or targets.class_id is None: @@ -171,12 +190,18 @@ class Recall(Metric["RecallResult"]): "and targets." ) if len(predictions) == 0: + target_class_ids = np.asarray(targets.class_id, dtype=np.int32)[ + target_size_mask + ] + if len(target_class_ids) == 0: + continue stats.append( ( + np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0, iou_thresholds.size), dtype=bool), np.zeros((0,), dtype=np.float32), np.zeros((0,), dtype=int), - targets.class_id, + target_class_ids, ) ) @@ -211,15 +236,48 @@ class Recall(Metric["RecallResult"]): "Unsupported metric target for IoU calculation" ) - matches = self._match_detection_batch( - prediction_class_ids, - target_class_ids, - iou, - iou_thresholds, + matches, matched_target_indices = ( + _match_detection_batch_with_target_indices( + prediction_class_ids, + target_class_ids, + iou, + iou_thresholds, + ) ) + ignored_matches = np.zeros_like(matches, dtype=bool) + if size_category != ObjectSizeCategory.ANY: + valid_target_match = matched_target_indices >= 0 + matched_scored_target = np.zeros_like(matches, dtype=bool) + if np.any(valid_target_match): + matched_scored_target[valid_target_match] = ( + target_size_mask[ + matched_target_indices[valid_target_match] + ] + ) + prediction_scored = ( + prediction_size_mask[:, None] | matched_scored_target + ) + ignored_matches = ~prediction_scored | ( + valid_target_match & ~matched_scored_target + ) + prediction_keep = np.any(~ignored_matches, axis=1) + matches = ( + matches[prediction_keep] + & matched_scored_target[prediction_keep] + ) + ignored_matches = ignored_matches[prediction_keep] + prediction_confidence = prediction_confidence[prediction_keep] + prediction_class_ids = prediction_class_ids[prediction_keep] + target_class_ids = target_class_ids[target_size_mask] + if ( + len(prediction_class_ids) == 0 + and len(target_class_ids) == 0 + ): + continue stats.append( ( matches, + ignored_matches, prediction_confidence, prediction_class_ids, target_class_ids, @@ -259,6 +317,7 @@ class Recall(Metric["RecallResult"]): def _compute_recall_for_classes( self, matches: npt.NDArray[np.bool_], + ignored_matches: npt.NDArray[np.bool_], prediction_confidence: npt.NDArray[np.float32], prediction_class_ids: npt.NDArray[np.int32], true_class_ids: npt.NDArray[np.int32], @@ -269,12 +328,13 @@ class Recall(Metric["RecallResult"]): ]: sorted_indices = np.argsort(-prediction_confidence) matches = matches[sorted_indices] + ignored_matches = ignored_matches[sorted_indices] prediction_class_ids = prediction_class_ids[sorted_indices] unique_classes, class_counts = np.unique(true_class_ids, return_counts=True) # Shape: PxTh,P,C,C -> CxThx3 confusion_matrix = self._compute_confusion_matrix( - matches, prediction_class_ids, unique_classes, class_counts + matches, ignored_matches, prediction_class_ids, unique_classes, class_counts ) # Shape: CxThx3 -> CxTh @@ -288,7 +348,15 @@ class Recall(Metric["RecallResult"]): recall_scores = self._compute_recall(confusion_matrix_merged) elif self.averaging_method == AveragingMethod.WEIGHTED: class_counts = class_counts.astype(np.float32) - recall_scores = np.average(recall_per_class, axis=0, weights=class_counts) + if class_counts.sum() == 0: + # No ground-truth support (e.g. only false-positive classes, or a + # size bucket with predictions but no targets): weighting is + # undefined, so report 0 as the empty case did before. + recall_scores = np.zeros(recall_per_class.shape[1]) + else: + recall_scores = np.average( + recall_per_class, axis=0, weights=class_counts + ) return recall_scores, recall_per_class, unique_classes @@ -299,24 +367,15 @@ class Recall(Metric["RecallResult"]): iou: npt.NDArray[np.float32], iou_thresholds: npt.NDArray[np.float32], ) -> npt.NDArray[np.bool_]: - num_predictions, num_iou_levels = ( - predictions_classes.shape[0], - iou_thresholds.shape[0], + result_correct, _ = _match_detection_batch_with_target_indices( + predictions_classes, target_classes, iou, iou_thresholds ) - correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) - correct_class = target_classes[:, None] == predictions_classes - - for i, iou_level in enumerate(iou_thresholds): - matched_indices = np.where((iou >= iou_level) & correct_class) - - for t, p in _greedy_match(iou, matched_indices): - correct[p, i] = True - result_correct: npt.NDArray[np.bool_] = correct return result_correct @staticmethod def _compute_confusion_matrix( sorted_matches: npt.NDArray[np.bool_], + sorted_ignored_matches: npt.NDArray[np.bool_], sorted_prediction_class_ids: npt.NDArray[np.int32], unique_classes: npt.NDArray[np.integer], class_counts: npt.NDArray[np.integer], @@ -330,6 +389,8 @@ class Recall(Metric["RecallResult"]): Args: sorted_matches: shape (P, Th), that is True if the prediction is a true positive at the given IoU threshold. + sorted_ignored_matches: shape (P, Th), that is True + if the prediction should not affect the given IoU threshold. sorted_prediction_class_ids: shape (P,), containing the class id for each prediction. unique_classes: shape (C,), containing the unique @@ -359,11 +420,13 @@ class Recall(Metric["RecallResult"]): false_negatives = np.full(num_thresholds, num_true) elif num_true == 0: true_positives = np.zeros(num_thresholds) - false_positives = np.full(num_thresholds, num_predictions) + false_positives = (~sorted_ignored_matches[is_class]).sum(0) false_negatives = np.zeros(num_thresholds) else: true_positives = sorted_matches[is_class].sum(0) - false_positives = (1 - sorted_matches[is_class]).sum(0) + false_positives = ( + ~sorted_matches[is_class] & ~sorted_ignored_matches[is_class] + ).sum(0) false_negatives = num_true - true_positives confusion_matrix[class_idx] = np.stack( [true_positives, false_positives, false_negatives], axis=1 @@ -419,6 +482,11 @@ class Recall(Metric["RecallResult"]): if detections.mask is not None: # detections.mask is NDArray[bool] | CompactMask; return as-is. return detections.mask + if len(detections) > 0: + raise ValueError( + "Recall with `MetricTarget.MASKS` requires detections to " + "include masks." + ) return self._make_empty_content() if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: obb = detections.data.get(ORIENTED_BOX_COORDINATES) diff --git a/src/supervision/metrics/utils/matching.py b/src/supervision/metrics/utils/matching.py index 43f1f4fb..dc72234c 100644 --- a/src/supervision/metrics/utils/matching.py +++ b/src/supervision/metrics/utils/matching.py @@ -31,3 +31,26 @@ def _greedy_match( matched_targets.add(t) matched_preds.add(p) yield t, p + + +def _match_detection_batch_with_target_indices( + predictions_classes: npt.NDArray[np.int32], + target_classes: npt.NDArray[np.int32], + iou: npt.NDArray[np.float32], + iou_thresholds: npt.NDArray[np.float32], +) -> tuple[npt.NDArray[np.bool_], npt.NDArray[np.int32]]: + """Match predictions to targets and retain target indices per IoU threshold.""" + num_predictions = predictions_classes.shape[0] + num_iou_levels = iou_thresholds.shape[0] + correct = np.zeros((num_predictions, num_iou_levels), dtype=bool) + matched_targets = np.full((num_predictions, num_iou_levels), -1, dtype=np.int32) + correct_class = target_classes[:, None] == predictions_classes + + for i, iou_level in enumerate(iou_thresholds): + matched_indices = np.where((iou >= iou_level) & correct_class) + + for target_idx, prediction_idx in _greedy_match(iou, matched_indices): + correct[prediction_idx, i] = True + matched_targets[prediction_idx, i] = target_idx + + return correct, matched_targets diff --git a/tests/classification/test_core.py b/tests/classification/test_core.py index 4fd2518d..55facb6e 100644 --- a/tests/classification/test_core.py +++ b/tests/classification/test_core.py @@ -121,3 +121,22 @@ def test_from_timm_softmaxes_logits() -> None: result.confidence, _MockTensor(logits).softmax(dim=-1).numpy()[0] ) assert np.isclose(np.sum(result.confidence), 1.0) + + +def test_classifications_compare_numpy_fields_by_value() -> None: + """Classifications equality handles NumPy arrays and confidence values.""" + left = Classifications( + class_id=np.array([0, 1], dtype=np.int_), + confidence=np.array([0.25, 0.75], dtype=np.float32), + ) + right = Classifications( + class_id=np.array([0, 1], dtype=np.int_), + confidence=np.array([0.25, 0.75], dtype=np.float32), + ) + different = Classifications( + class_id=np.array([0, 1], dtype=np.int_), + confidence=np.array([0.25, 0.5], dtype=np.float32), + ) + + assert left == right + assert left != different diff --git a/tests/dataset/test_core.py b/tests/dataset/test_core.py index 2a223e45..5fe9da16 100644 --- a/tests/dataset/test_core.py +++ b/tests/dataset/test_core.py @@ -5,7 +5,12 @@ import numpy as np import numpy.typing as npt import pytest -from supervision import ClassificationDataset, DetectionDataset, Detections +from supervision import ( + ClassificationDataset, + Classifications, + DetectionDataset, + Detections, +) from supervision.config import CLASS_NAME_DATA_FIELD from supervision.utils.internal import SupervisionWarnings from tests.helpers import _create_detections, create_yolo_dataset @@ -436,6 +441,49 @@ class TestDetectionDatasetInMemoryImages: assert ds_a != ds_b +class TestDatasetEqualityContracts: + """Dataset equality must respect class order and NumPy-backed annotations.""" + + def test_detection_dataset_class_order_matters(self) -> None: + """DetectionDataset equality is sensitive to the ordered class list.""" + ds_a = DetectionDataset(classes=["cat", "dog"], images=[], annotations={}) + ds_b = DetectionDataset(classes=["dog", "cat"], images=[], annotations={}) + + assert ds_a != ds_b + + def test_classification_dataset_numpy_annotations(self) -> None: + """ClassificationDataset equality handles multi-value NumPy annotations.""" + annotations = { + "img.png": Classifications( + class_id=np.array([0, 1], dtype=np.int_), + confidence=np.array([0.25, 0.75], dtype=np.float32), + ) + } + ds_a = ClassificationDataset( + classes=["cat", "dog"], + images=["img.png"], + annotations=annotations, + ) + ds_b = ClassificationDataset( + classes=["cat", "dog"], + images=["img.png"], + annotations={ + "img.png": Classifications( + class_id=np.array([0, 1], dtype=np.int_), + confidence=np.array([0.25, 0.75], dtype=np.float32), + ) + }, + ) + ds_c = ClassificationDataset( + classes=["dog", "cat"], + images=["img.png"], + annotations=annotations, + ) + + assert ds_a == ds_b + assert ds_a != ds_c + + class TestDetectionDatasetExportCollisions: """Regression tests for the basename-collision guard on export (DAT-04).""" diff --git a/tests/metrics/test_f1_score.py b/tests/metrics/test_f1_score.py index 33874494..163079c1 100644 --- a/tests/metrics/test_f1_score.py +++ b/tests/metrics/test_f1_score.py @@ -160,6 +160,23 @@ class TestF1Score: assert result.f1_50 == 0.0 assert result.f1_75 == 0.0 + def test_medium_bucket_scores_target_matched_small_prediction(self) -> None: + """Medium-object F1 keeps valid matches even if the prediction is small.""" + predictions = Detections( + xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32), + class_id=np.array([0]), + ) + + result = F1Score().update(predictions, targets).compute() + + assert result.medium_objects is not None + assert result.medium_objects.f1_50 == 1.0 + def test_false_positives_on_background_image_counted(self): """Predictions on an image with no targets must count as false positives.""" predictions_with_gt = Detections( diff --git a/tests/metrics/test_mean_average_recall.py b/tests/metrics/test_mean_average_recall.py index d82489df..aedfe040 100644 --- a/tests/metrics/test_mean_average_recall.py +++ b/tests/metrics/test_mean_average_recall.py @@ -502,6 +502,33 @@ def test_empty_inputs_keep_max_detection_axis() -> None: assert result.matched_classes.shape == (0,) +def test_medium_bucket_scores_target_matched_small_prediction() -> None: + """Medium-object mAR keeps valid matches even if the prediction is small.""" + predictions = Detections( + xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + targets = Detections( + xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + + result = ( + MeanAverageRecall(metric_target=MetricTarget.BOXES) + .update( + [predictions], + [targets], + ) + .compute() + ) + + assert result.medium_objects is not None + assert result.medium_objects.mAR_at_1 == pytest.approx(0.9) + assert result.medium_objects.mAR_at_10 == pytest.approx(0.9) + assert result.medium_objects.mAR_at_100 == pytest.approx(0.9) + + @pytest.mark.parametrize( "missing_attribute", ["predictions_class_id", "targets_class_id", "predictions_confidence"], diff --git a/tests/metrics/test_precision.py b/tests/metrics/test_precision.py index c1317eb0..72bcd95a 100644 --- a/tests/metrics/test_precision.py +++ b/tests/metrics/test_precision.py @@ -115,6 +115,23 @@ class TestPrecision: assert result.precision_at_50 == 0.0 assert result.precision_at_75 == 0.0 + def test_medium_bucket_scores_target_matched_small_prediction(self) -> None: + """Medium-object precision keeps a small matched prediction in the score.""" + predictions = Detections( + xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32), + class_id=np.array([0]), + ) + + result = Precision().update(predictions, targets).compute() + + assert result.medium_objects is not None + assert result.medium_objects.precision_at_50 == 1.0 + def test_false_positives_on_background_image_counted(self): """Predictions on an image with no targets must count as false positives.""" predictions_with_gt = Detections( diff --git a/tests/metrics/test_recall.py b/tests/metrics/test_recall.py index 225dc3e7..1b5ec7e7 100644 --- a/tests/metrics/test_recall.py +++ b/tests/metrics/test_recall.py @@ -149,6 +149,23 @@ class TestRecall: assert result.recall_at_50 == 0.0 assert result.recall_at_75 == 0.0 + def test_medium_bucket_scores_target_matched_small_prediction(self) -> None: + """Medium-object recall keeps valid matches even if the prediction is small.""" + predictions = Detections( + xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32), + class_id=np.array([0]), + ) + + result = Recall().update(predictions, targets).compute() + + assert result.medium_objects is not None + assert result.medium_objects.recall_at_50 == 1.0 + def test_single_class_missed_detections( self, detections_50_50, targets_two_objects_class_0 ): diff --git a/tests/metrics/test_size_bucket_regressions.py b/tests/metrics/test_size_bucket_regressions.py new file mode 100644 index 00000000..18fb10a6 --- /dev/null +++ b/tests/metrics/test_size_bucket_regressions.py @@ -0,0 +1,177 @@ +import numpy as np +import pytest + +from supervision.detection.core import Detections +from supervision.metrics import ( + F1Score, + MeanAverageRecall, + MetricTarget, + Precision, + Recall, +) + + +@pytest.mark.parametrize( + ("metric_cls", "bucket_attr", "score_attrs", "expected"), + [ + pytest.param( + Precision, + "medium_objects", + ("precision_at_50", "precision_at_75"), + (1.0, 1.0), + id="precision", + ), + pytest.param( + Recall, + "medium_objects", + ("recall_at_50", "recall_at_75"), + (1.0, 1.0), + id="recall", + ), + pytest.param( + F1Score, + "medium_objects", + ("f1_50", "f1_75"), + (1.0, 1.0), + id="f1", + ), + pytest.param( + MeanAverageRecall, + "medium_objects", + ("mAR_at_1", "mAR_at_10", "mAR_at_100"), + (0.6, 0.6, 0.6), + id="mar", + ), + ], +) +def test_size_bucket_match_is_not_stolen( + metric_cls, bucket_attr, score_attrs, expected +): + """Bucketed metrics must keep the in-bucket match instead of stealing it.""" + predictions = Detections( + xyxy=np.array([[0, 0, 90, 90]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + targets = Detections( + xyxy=np.array( + [[0, 0, 80, 80], [0, 0, 100, 100]], + dtype=np.float32, + ), + class_id=np.array([0, 0], dtype=np.int32), + ) + + result = ( + metric_cls(metric_target=MetricTarget.BOXES) + .update(predictions, targets) + .compute() + ) + + bucket_result = getattr(result, bucket_attr) + assert bucket_result is not None + for score_attr, score_expected in zip(score_attrs, expected, strict=True): + assert getattr(bucket_result, score_attr) == pytest.approx(score_expected) + + +def test_small_bucket_mar_returns_zero_without_bucket_targets() -> None: + """Bucketed mAR must return zeros instead of NaN when there is no support.""" + predictions = Detections( + xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32), + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + targets = Detections( + xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + + result = ( + MeanAverageRecall(metric_target=MetricTarget.BOXES) + .update(predictions, targets) + .compute() + ) + + assert result.small_objects is not None + np.testing.assert_allclose(result.small_objects.recall_scores, np.zeros(3)) + assert result.small_objects.mAR_at_1 == 0.0 + assert result.small_objects.mAR_at_10 == 0.0 + assert result.small_objects.mAR_at_100 == 0.0 + + +def test_medium_bucket_mar_counts_global_rank_budget() -> None: + """Bucketed mAR must count out-of-bucket predictions against top-K.""" + predictions = Detections( + xyxy=np.array( + [[0, 0, 150, 150], [0, 0, 80, 80]], + dtype=np.float32, + ), + confidence=np.array([0.95, 0.90], dtype=np.float32), + class_id=np.array([0, 0], dtype=np.int32), + ) + targets = Detections( + xyxy=np.array([[0, 0, 80, 80]], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + + result = ( + MeanAverageRecall(metric_target=MetricTarget.BOXES) + .update(predictions, targets) + .compute() + ) + + assert result.medium_objects is not None + assert result.medium_objects.mAR_at_1 == 0.0 + assert result.medium_objects.mAR_at_10 == 1.0 + assert result.medium_objects.mAR_at_100 == 1.0 + + +@pytest.mark.parametrize( + ("metric_cls", "missing_side"), + [ + pytest.param(Precision, "predictions", id="precision-predictions"), + pytest.param(Precision, "targets", id="precision-targets"), + pytest.param(Recall, "predictions", id="recall-predictions"), + pytest.param(Recall, "targets", id="recall-targets"), + pytest.param(F1Score, "predictions", id="f1-predictions"), + pytest.param(F1Score, "targets", id="f1-targets"), + pytest.param( + MeanAverageRecall, + "predictions", + id="mar-predictions", + ), + pytest.param(MeanAverageRecall, "targets", id="mar-targets"), + ], +) +def test_mask_target_requires_masks(metric_cls, missing_side) -> None: + """Mask-target metrics raise when either side omits masks.""" + box = np.array([[0, 0, 10, 10]], dtype=np.float32) + mask = np.zeros((1, 10, 10), dtype=bool) + mask[0, 2:8, 2:8] = True + + masked_predictions = Detections( + xyxy=box, + mask=mask, + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + masked_targets = Detections( + xyxy=box, + mask=mask, + class_id=np.array([0], dtype=np.int32), + ) + + predictions = masked_predictions + targets = masked_targets + if missing_side == "predictions": + predictions = Detections( + xyxy=box, + confidence=np.array([0.9], dtype=np.float32), + class_id=np.array([0], dtype=np.int32), + ) + else: + targets = Detections(xyxy=box, class_id=np.array([0], dtype=np.int32)) + + metric = metric_cls(metric_target=MetricTarget.MASKS) + + with pytest.raises(ValueError, match="requires detections to include masks"): + metric.update(predictions, targets).compute()