Files
7fc91180fe fix(metrics): track prediction-only classes in Recall (#2468)
#2331 made Precision and F1Score include classes that appear only in
predictions, and added regression tests to both. Recall was not touched, so
the line #2331 replaced is still there and the three metrics disagree about
which classes exist for identical input:

    precision.matched_classes -> [0 1]   precision_per_class (2, 10)
    recall.matched_classes    -> [0]     recall_per_class    (1, 10)
    f1.matched_classes        -> [0 1]

These read as parallel outputs, so zipping them silently truncates rather
than raising.

Recall for a class with no ground-truth instances is 0.0 rather than
undefined, which is what sklearn reports (it infers labels from the union of
y_true and y_pred) and what #2331 cited as its own standard. MICRO is
unchanged because an absent class contributes no false negatives, and
WEIGHTED is unchanged because its ground-truth support is zero. MACRO does
change, and the changelog says so.

Also of note: recall.py already carried #2331's WEIGHTED zero-support guard,
whose comment refers to 'only false-positive classes'. That state could not
arise in recall.py, because unique_classes came from ground truth alone. The
guard was propagated; the union that gives it meaning was not.

Addresses the review on #2468. Building the class union inside
_compute_recall_for_classes only covers samples that reach it, and samples with
predictions but no targets are skipped earlier in _compute. So matched_classes
could still disagree with Precision and F1Score for list inputs containing a
background image, which is the exact invariant the new test asserts.

Before, for one normal sample plus one background image predicting class 2:

    precision.matched_classes -> [0 2]
    recall.matched_classes    -> [0]

Recall now handles len(targets) == 0 and len(predictions) > 0 the way Precision
does. No recall value changes, since a background image produces no false
negatives; only the tracked class set does.

* test: cover Recall bg-image size-bucket, dup & non-contiguous ids
* docs: strengthen Recall changelog migration note
* docs+perf: Recall doctest example; dedupe-then-union micro-opt

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
2026-08-03 20:57:05 +02:00

606 lines
24 KiB
Python

import numpy as np
import pytest
from supervision.detection.compact_mask import CompactMask
from supervision.detection.core import Detections
from supervision.metrics.core import AveragingMethod, MetricTarget
from supervision.metrics.f1_score import F1Score
from supervision.metrics.precision import Precision
from supervision.metrics.recall import Recall
from tests.helpers import assert_almost_equal
class TestRecall:
@pytest.fixture
def predictions_multiple_classes(self):
return Detections(
xyxy=np.array(
[
[10, 10, 50, 50], # class 0, matches first target
[200, 200, 240, 240], # class 1, matches target
],
dtype=np.float32,
),
confidence=np.array([0.9, 0.8]),
class_id=np.array([0, 1]),
)
@pytest.fixture
def targets_multiple_classes(self):
return Detections(
xyxy=np.array(
[
[10, 10, 50, 50], # class 0, matched
[60, 60, 100, 100], # class 0, missed
[200, 200, 240, 240], # class 1, matched
],
dtype=np.float32,
),
class_id=np.array([0, 0, 1]),
)
def test_initialization_default(self):
"""Test that Recall can be initialized with default parameters"""
metric = Recall()
assert metric._metric_target == MetricTarget.BOXES
assert metric.averaging_method == AveragingMethod.WEIGHTED
assert metric._predictions_list == []
assert metric._targets_list == []
def test_initialization_custom(self):
"""Test that Recall can be initialized with custom parameters"""
metric = Recall(
metric_target=MetricTarget.MASKS,
averaging_method=AveragingMethod.MACRO,
)
assert metric._metric_target == MetricTarget.MASKS
assert metric.averaging_method == AveragingMethod.MACRO
def test_mask_content_preserves_compact_mask(self) -> None:
"""CompactMask inputs stay compact for mask IoU."""
dense_mask = np.zeros((1, 4, 5), dtype=bool)
dense_mask[0, 1:3, 1:4] = True
xyxy = np.array([[1, 1, 4, 3]], dtype=np.float64)
compact_mask = CompactMask.from_dense(
dense_mask, xyxy=xyxy, image_shape=dense_mask.shape[1:]
)
detections = Detections(xyxy=xyxy, mask=compact_mask)
metric = Recall(metric_target=MetricTarget.MASKS)
content = metric._detections_content(detections)
assert content is compact_mask
def test_compute_with_compact_mask_matches_dense(self) -> None:
"""Recall.compute() yields same recall_at_50 for CompactMask and dense."""
masks = np.zeros((1, 50, 50), dtype=bool)
masks[0, 10:20, 10:20] = True
xyxy = np.array([[10, 10, 19, 19]], dtype=np.float64)
cm = CompactMask.from_dense(masks, xyxy, image_shape=(50, 50))
det_dense = Detections(
xyxy=xyxy, mask=masks, confidence=np.array([0.9]), class_id=np.array([0])
)
det_compact = Detections(
xyxy=xyxy, mask=cm, confidence=np.array([0.9]), class_id=np.array([0])
)
metric = Recall(metric_target=MetricTarget.MASKS)
r_dense = metric.update(det_dense, det_dense).compute()
metric.reset()
r_compact = metric.update(det_compact, det_compact).compute()
assert r_dense.recall_at_50 == pytest.approx(r_compact.recall_at_50)
def test_reset(self, dummy_prediction):
"""Test that reset() clears all stored data"""
metric = Recall()
# Add some dummy data
metric.update(dummy_prediction, dummy_prediction)
# Verify data was added
assert len(metric._predictions_list) == 1
assert len(metric._targets_list) == 1
# Reset and verify lists are empty
metric.reset()
assert metric._predictions_list == []
assert metric._targets_list == []
def test_perfect_match(self, detections_50_50, targets_50_50):
"""Test recall with perfect matching predictions and targets"""
metric = Recall()
result = metric.update(detections_50_50, targets_50_50).compute()
# Perfect match should give recall = 1.0
# TP = 1, FN = 0 -> recall = TP / (TP + FN) = 1 / 1 = 1.0
assert result.recall_at_50 == 1.0
assert result.recall_at_75 == 1.0
assert len(result.matched_classes) == 1
assert result.matched_classes[0] == 0
def test_no_overlap(self, predictions_no_overlap, targets_no_overlap):
"""Test recall with predictions that don't overlap with targets"""
metric = Recall()
result = metric.update(predictions_no_overlap, targets_no_overlap).compute()
# No overlap means no TP, only FN
# TP = 0, FN = 1 -> recall = TP / (TP + FN) = 0 / 1 = 0.0
assert result.recall_at_50 == 0.0
assert result.recall_at_75 == 0.0
@pytest.mark.parametrize(
("method", "expected"),
[
pytest.param(
AveragingMethod.MICRO, 1.0, id="micro-absent-class-adds-no-fn"
),
pytest.param(
AveragingMethod.MACRO, 0.5, id="macro-includes-absent-class-at-zero"
),
pytest.param(
AveragingMethod.WEIGHTED,
1.0,
id="weighted-absent-class-has-no-support-by-design",
),
],
)
def test_absent_class_predictions_are_tracked(self, method, expected):
"""A class predicted but never present in the targets is still tracked.
Recall for such a class is 0.0 rather than undefined, which is what sklearn
reports and what Precision and F1Score already do here. MICRO is unchanged
because an absent class contributes no false negatives, and WEIGHTED is
unchanged because its ground-truth support is zero.
"""
predictions = Detections(
xyxy=np.array(
[[0, 0, 10, 10], [100, 0, 110, 10], [120, 0, 130, 10]], np.float32
),
class_id=np.array([0, 1, 1]), # class 1 never appears in the targets
confidence=np.array([0.9, 0.8, 0.7]),
)
targets = Detections(
xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32),
class_id=np.array([0]),
)
metric = Recall(averaging_method=method)
result = metric.update(predictions, targets).compute()
assert result.recall_at_50 == expected
assert list(result.matched_classes) == [0, 1]
def test_tracked_classes_match_precision_and_f1(self):
"""The three metrics must agree on which classes exist for the same data.
They return `matched_classes` and a `*_per_class` array that read as parallel
outputs. When the class sets diverge, zipping them silently truncates instead
of raising.
"""
predictions = Detections(
xyxy=np.array([[0, 0, 10, 10], [100, 0, 110, 10]], dtype=np.float32),
class_id=np.array([0, 1]),
confidence=np.array([0.9, 0.8]),
)
targets = Detections(
xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32),
class_id=np.array([0]),
)
recall = Recall().update(predictions, targets).compute()
precision = Precision().update(predictions, targets).compute()
f1 = F1Score().update(predictions, targets).compute()
assert list(recall.matched_classes) == list(precision.matched_classes)
assert list(recall.matched_classes) == list(f1.matched_classes)
assert (
recall.recall_per_class.shape[0] == precision.precision_per_class.shape[0]
)
@pytest.mark.parametrize(
("averaging_method", "expected_recall_at_50"),
[
pytest.param(
AveragingMethod.WEIGHTED,
1.0,
id="weighted-background-class-has-no-support",
),
pytest.param(
AveragingMethod.MICRO,
1.0,
id="micro-background-class-adds-no-fn",
),
pytest.param(
AveragingMethod.MACRO,
0.5,
id="macro-background-class-drags-average-down",
),
],
)
def test_tracked_classes_match_precision_and_f1_with_background_images(
self, averaging_method: AveragingMethod, expected_recall_at_50: float
) -> None:
"""The class sets must still agree when a sample has predictions and no targets.
A background image produces no false negatives, so no recall value changes
under WEIGHTED or MICRO, but its predicted classes still have to be tracked.
Building the class union only inside the targets-present path leaves them
out, and the three metrics disagree again for list inputs that contain one.
MACRO is where the background class's 0.0 recall placeholder visibly shifts
the aggregate, since it is averaged unweighted across classes.
"""
with_targets_pred = Detections(
xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32),
class_id=np.array([0]),
confidence=np.array([0.9]),
)
with_targets_gt = Detections(
xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32),
class_id=np.array([0]),
)
background_pred = Detections(
xyxy=np.array([[50, 50, 60, 60]], dtype=np.float32),
class_id=np.array([2]), # class 2 exists only on a target-less sample
confidence=np.array([0.8]),
)
preds = [with_targets_pred, background_pred]
gts = [with_targets_gt, Detections.empty()]
recall = Recall(averaging_method=averaging_method).update(preds, gts).compute()
precision = Precision().update(preds, gts).compute()
f1 = F1Score().update(preds, gts).compute()
assert list(recall.matched_classes) == [0, 2]
assert list(recall.matched_classes) == list(precision.matched_classes)
assert list(recall.matched_classes) == list(f1.matched_classes)
assert (
recall.recall_per_class.shape[0] == precision.precision_per_class.shape[0]
)
assert recall.recall_at_50 == pytest.approx(expected_recall_at_50)
def test_background_image_size_bucket_filters_predictions_by_size(self) -> None:
"""Size buckets restrict background-image prediction-only classes by size.
A background image (empty targets) with predictions of different sizes must
only surface in the size bucket matching that size; a bucket left with zero
predictions after size filtering must stay empty rather than error.
"""
predictions = Detections(
xyxy=np.array(
[
[0, 0, 10, 10], # area 100 (Small)
[0, 0, 200, 200], # area 40000 (Large)
],
dtype=np.float32,
),
confidence=np.array([0.9, 0.8]),
class_id=np.array([1, 2]),
)
targets = Detections.empty()
result = Recall().update(predictions, targets).compute()
assert result.small_objects is not None
assert list(result.small_objects.matched_classes) == [1]
assert result.small_objects.recall_per_class.shape == (1, 10)
assert result.large_objects is not None
assert list(result.large_objects.matched_classes) == [2]
assert result.large_objects.recall_per_class.shape == (1, 10)
# Neither prediction is Medium: size filtering leaves zero predictions, so
# the `continue` path is hit and the sample contributes nothing to this bucket.
assert result.medium_objects is not None
assert list(result.medium_objects.matched_classes) == []
assert result.medium_objects.recall_per_class.shape == (0, 10)
def test_multiple_background_image_samples_accumulate_classes(self) -> None:
"""Two background-image samples in one list input union their classes.
Prediction-only classes from separate background-only samples must all be
tracked together, not just the first sample's class.
"""
background_pred_1 = Detections(
xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32),
class_id=np.array([5]),
confidence=np.array([0.9]),
)
background_pred_2 = Detections(
xyxy=np.array([[20, 20, 30, 30]], dtype=np.float32),
class_id=np.array([9]),
confidence=np.array([0.7]),
)
preds = [background_pred_1, background_pred_2]
gts = [Detections.empty(), Detections.empty()]
result = Recall().update(preds, gts).compute()
assert list(result.matched_classes) == [5, 9]
assert result.recall_per_class.shape == (2, 10)
assert result.recall_at_50 == 0.0
def test_background_image_duplicate_class_ids_deduplicated(self) -> None:
"""Repeated class ids in one background-image sample collapse to a set."""
predictions = Detections(
xyxy=np.array(
[[0, 0, 10, 10], [20, 20, 30, 30], [40, 40, 50, 50]], dtype=np.float32
),
confidence=np.array([0.9, 0.8, 0.7]),
class_id=np.array([2, 2, 3]),
)
targets = Detections.empty()
result = Recall().update(predictions, targets).compute()
assert list(result.matched_classes) == [2, 3]
assert result.recall_per_class.shape == (2, 10)
assert result.recall_at_50 == 0.0
def test_non_contiguous_class_ids_align_by_value_not_index(self) -> None:
"""`matched_classes` rows align to class-id values, not positional order.
Large, non-contiguous class ids are sorted numerically by `np.unique` /
`searchsorted`; a positional-index bug would misalign the prediction-only
class's zero recall with the matched class's row.
"""
predictions = Detections(
xyxy=np.array(
[
[100, 100, 140, 140], # class 1000, no matching target
[10, 10, 50, 50], # class 5, matches first target exactly
],
dtype=np.float32,
),
confidence=np.array([0.95, 0.9]),
class_id=np.array([1000, 5]),
)
targets = Detections(
xyxy=np.array(
[
[10, 10, 50, 50], # class 5, matched
[200, 200, 240, 240], # class 5, missed
],
dtype=np.float32,
),
class_id=np.array([5, 5]),
)
result = Recall().update(predictions, targets).compute()
assert list(result.matched_classes) == [5, 1000]
assert result.recall_per_class.shape == (2, 10)
assert np.all(result.recall_per_class[0] == 0.5) # class 5: TP=1, FN=1
assert np.all(result.recall_per_class[1] == 0.0) # class 1000: no targets
assert result.recall_at_50 == pytest.approx(0.5)
def test_empty_predictions(self, targets_50_50):
"""Test recall with empty predictions but existing targets"""
predictions = Detections.empty()
metric = Recall()
result = metric.update(predictions, targets_50_50).compute()
# No predictions means TP = 0, FN = 1 -> recall = 0 / 1 = 0.0
assert result.recall_at_50 == 0.0
assert result.recall_at_75 == 0.0
def test_empty_targets(self, detections_50_50):
"""Test recall with predictions but no targets"""
targets = Detections.empty()
metric = Recall()
result = metric.update(detections_50_50, targets).compute()
# No targets means TP = 0, FN = 0 -> recall = 0 / 0 = 0
assert result.recall_at_50 == 0.0
assert result.recall_at_75 == 0.0
def test_medium_bucket_scores_target_matched_small_prediction(self) -> None:
"""Medium-object recall keeps valid matches even if the prediction is small."""
predictions = Detections(
xyxy=np.array([[0, 0, 31, 31]], dtype=np.float32),
confidence=np.array([0.9], dtype=np.float32),
class_id=np.array([0]),
)
targets = Detections(
xyxy=np.array([[0, 0, 32, 32]], dtype=np.float32),
class_id=np.array([0]),
)
result = Recall().update(predictions, targets).compute()
assert result.medium_objects is not None
assert result.medium_objects.recall_at_50 == 1.0
def test_single_class_missed_detections(
self, detections_50_50, targets_two_objects_class_0
):
"""Test recall calculation with some missed detections"""
metric = Recall()
result = metric.update(detections_50_50, targets_two_objects_class_0).compute()
# TP = 1 (first target matched), FN = 1 (second target missed)
# recall = TP / (TP + FN) = 1 / 2 = 0.5
assert_almost_equal(result.recall_at_50, 0.5)
assert_almost_equal(result.recall_at_75, 0.5)
def test_multiple_classes(
self, predictions_multiple_classes, targets_multiple_classes
):
"""Test recall calculation for multiple classes"""
metric = Recall()
result = metric.update(
predictions_multiple_classes, targets_multiple_classes
).compute()
# Class 0: TP=1, FN=1 -> recall=0.5 (weight=2)
# Class 1: TP=1, FN=0 -> recall=1.0 (weight=1)
# Weighted avg: (2*0.5 + 1*1.0) / (2+1) = 2.0/3 = 0.6667
expected_recall = (2 * 0.5 + 1 * 1.0) / (2 + 1)
assert_almost_equal(result.recall_at_50, expected_recall)
assert_almost_equal(result.recall_at_75, expected_recall)
assert len(result.matched_classes) == 2
assert 0 in result.matched_classes
assert 1 in result.matched_classes
def test_different_iou_thresholds(self, predictions_iou_064, targets_iou_064):
"""Test recall at different IoU thresholds"""
metric = Recall()
result = metric.update(predictions_iou_064, targets_iou_064).compute()
# IoU = 0.64 > 0.5 but < 0.75
# Should match at IoU 0.5 but not at 0.75
assert result.recall_at_50 == 1.0 # TP=1, FN=0
assert result.recall_at_75 == 0.0 # TP=0, FN=1
def test_confidence_ranking(self, predictions_confidence_ranking, targets_50_50):
"""Test that higher confidence predictions are preferred for matching"""
metric = Recall()
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
# Target should be matched (by higher confidence prediction)
# TP = 1, FN = 0 -> recall = 1.0
assert result.recall_at_50 == 1.0
def test_multiple_predictions_one_target(
self, predictions_confidence_ranking, targets_50_50
):
"""Test recall when multiple predictions compete for one target"""
metric = Recall()
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
# Target should be matched exactly once
# TP = 1, FN = 0 -> recall = 1.0
assert result.recall_at_50 == 1.0
def test_list_inputs(
self, detections_50_50, targets_50_50, prediction_class_1, target_class_1
):
"""Test recall with list inputs"""
metric = Recall()
result = metric.update(
[detections_50_50, prediction_class_1], [targets_50_50, target_class_1]
).compute()
# Perfect matches for both
assert result.recall_at_50 == 1.0
assert result.recall_at_75 == 1.0
def test_mismatched_list_lengths(self, detections_50_50, targets_50_50):
"""Test that mismatched prediction/target list lengths raise error"""
metric = Recall()
# Should raise ValueError for mismatched lengths
with pytest.raises(ValueError, match="number of predictions"):
metric.update([detections_50_50], [targets_50_50, targets_50_50])
@pytest.mark.parametrize(
"missing_attribute",
["predictions_class_id", "targets_class_id", "predictions_confidence"],
)
def test_compute_value_error_for_missing_required_fields(
self, missing_attribute
) -> None:
"""Test compute raises ValueError when required fields are missing."""
metric = Recall()
boxes = np.array([[10, 10, 50, 50]], dtype=np.float32)
class_id = np.array([0], dtype=np.int32)
confidence = np.array([0.9], dtype=np.float32)
predictions = Detections(
xyxy=boxes,
confidence=confidence,
class_id=class_id,
)
targets = Detections(
xyxy=boxes,
class_id=class_id,
)
if missing_attribute == "predictions_class_id":
predictions = Detections(
xyxy=boxes,
confidence=confidence,
)
elif missing_attribute == "targets_class_id":
targets = Detections(xyxy=boxes)
else:
predictions = Detections(
xyxy=boxes,
class_id=class_id,
)
with pytest.raises(ValueError, match="Recall metric requires"):
metric.update(predictions, targets).compute()
@pytest.mark.parametrize(
"averaging_method",
[AveragingMethod.MACRO, AveragingMethod.MICRO, AveragingMethod.WEIGHTED],
)
def test_averaging_methods(self, averaging_method, detections_50_50, targets_50_50):
"""Test different averaging methods"""
metric = Recall(averaging_method=averaging_method)
result = metric.update(detections_50_50, targets_50_50).compute()
# Perfect match should give 1.0 regardless of averaging method
assert result.recall_at_50 == 1.0
assert result.averaging_method == averaging_method
def test_macro_averaging(self):
"""Test MACRO averaging with specific example"""
# Class 0: 1/2 targets matched -> recall = 0.5
# Class 1: 1/1 targets matched -> recall = 1.0
# Macro average: (0.5 + 1.0) / 2 = 0.75
predictions = Detections(
xyxy=np.array(
[
[10, 10, 50, 50], # matches class 0 target 1
[200, 200, 240, 240], # matches class 1 target
],
dtype=np.float32,
),
confidence=np.array([0.9, 0.8]),
class_id=np.array([0, 1]),
)
targets = Detections(
xyxy=np.array(
[
[10, 10, 50, 50], # class 0, matched
[60, 60, 100, 100], # class 0, missed
[200, 200, 240, 240], # class 1, matched
],
dtype=np.float32,
),
class_id=np.array([0, 0, 1]),
)
metric = Recall(averaging_method=AveragingMethod.MACRO)
result = metric.update(predictions, targets).compute()
# Macro average: (0.5 + 1.0) / 2 = 0.75
assert result.recall_at_50 == 0.75
def test_greedy_matching_two_valid_pairs(self):
"""Greedy matching finds both TPs; np.unique style missed the second pair.
IoU matrix: [[1.0, 0.667], [0.333, 0.538]]. At iou>=0.5 the optimal
assignment is T0<->P0 and T1<->P1 (2 TPs, recall=1.0).
"""
preds = Detections(
xyxy=np.array([[40, 60, 380, 470], [108, 60, 448, 470]], dtype=np.float32),
confidence=np.array([0.95, 0.90]),
class_id=np.array([0, 0]),
)
targets = Detections(
xyxy=np.array([[40, 60, 380, 470], [210, 60, 550, 470]], dtype=np.float32),
class_id=np.array([0, 0]),
)
result = Recall().update(preds, targets).compute()
assert result.recall_at_50 == 1.0