add unit tests for precision, recall and f1_score metrics (#2119)
* add unit tests for precision, recall and f1_score metrics These 3 metrics had zero test coverage. Created comprehensive tests covering all main scenarios like perfect matches, partial overlaps, empty cases, multiple classes and different iou thresholds. All tests validate results mathematically and cover edge cases. Tests follow the existing code style and use pytest parametrize for different averaging methods. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jirka <6035284+Borda@users.noreply.github.com>
This commit is contained in:
Dhiego Pagotto
GitHub
parent
6c2c3f8862
commit
6e8e6fb276
|
|
@ -0,0 +1,377 @@
|
|||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from supervision.detection.core import Detections
|
||||
from supervision.metrics.core import AveragingMethod, MetricTarget
|
||||
from supervision.metrics.f1_score import F1Score
|
||||
from test.helpers import assert_almost_equal
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detections_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_prediction():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_two_objects_class_0():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50],
|
||||
[100, 100, 110, 110],
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0, matches target
|
||||
[60, 60, 100, 100], # class 1, matches target
|
||||
[120, 120, 130, 130], # class 1, false positive
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.9, 0.8, 0.7]),
|
||||
class_id=np.array([0, 1, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0
|
||||
[60, 60, 100, 100], # class 1
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_confidence_ranking():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # low confidence, perfect match
|
||||
[11, 11, 49, 49], # high confidence, good match
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.6, 0.9]), # second has higher confidence
|
||||
class_id=np.array([0, 0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prediction_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
class TestF1Score:
|
||||
def test_initialization_default(self):
|
||||
"""Test that F1Score can be initialized with default parameters"""
|
||||
metric = F1Score()
|
||||
assert metric._metric_target == MetricTarget.BOXES
|
||||
assert metric.averaging_method == AveragingMethod.WEIGHTED
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_initialization_custom(self):
|
||||
"""Test that F1Score can be initialized with custom parameters"""
|
||||
metric = F1Score(
|
||||
metric_target=MetricTarget.MASKS,
|
||||
averaging_method=AveragingMethod.MACRO,
|
||||
)
|
||||
assert metric._metric_target == MetricTarget.MASKS
|
||||
assert metric.averaging_method == AveragingMethod.MACRO
|
||||
|
||||
def test_reset(self, dummy_prediction):
|
||||
"""Test that reset() clears all stored data"""
|
||||
metric = F1Score()
|
||||
|
||||
# Add some dummy data
|
||||
metric.update(dummy_prediction, dummy_prediction)
|
||||
|
||||
# Verify data was added
|
||||
assert len(metric._predictions_list) == 1
|
||||
assert len(metric._targets_list) == 1
|
||||
|
||||
# Reset and verify lists are empty
|
||||
metric.reset()
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_perfect_match(self, detections_50_50, targets_50_50):
|
||||
"""Test F1 score with perfect matching predictions and targets"""
|
||||
metric = F1Score()
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give F1 = 1.0
|
||||
# TP = 1, FP = 0, FN = 0
|
||||
# Precision = TP / (TP + FP) = 1 / 1 = 1.0
|
||||
# Recall = TP / (TP + FN) = 1 / 1 = 1.0
|
||||
# F1 = 2 * (P * R) / (P + R) = 2 * 1.0 / 2 = 1.0
|
||||
assert result.f1_50 == 1.0
|
||||
assert result.f1_75 == 1.0
|
||||
assert len(result.matched_classes) == 1
|
||||
assert result.matched_classes[0] == 0
|
||||
|
||||
def test_no_overlap(self, predictions_no_overlap, targets_no_overlap):
|
||||
"""Test F1 score with predictions that don't overlap with targets"""
|
||||
metric = F1Score()
|
||||
result = metric.update(predictions_no_overlap, targets_no_overlap).compute()
|
||||
|
||||
# No overlap means TP=0, FP=1, FN=1
|
||||
# Precision = 0 / 1 = 0.0
|
||||
# Recall = 0 / 1 = 0.0
|
||||
# F1 = 2 * (0 * 0) / (0 + 0) = 0 / 0 = 0.0
|
||||
assert result.f1_50 == 0.0
|
||||
assert result.f1_75 == 0.0
|
||||
|
||||
def test_empty_predictions(self, targets_50_50):
|
||||
"""Test F1 score with empty predictions but existing targets"""
|
||||
predictions = Detections.empty()
|
||||
|
||||
metric = F1Score()
|
||||
result = metric.update(predictions, targets_50_50).compute()
|
||||
|
||||
# No predictions: TP=0, FP=0, FN=1
|
||||
# Precision = 0 / 0 = 0 (by convention)
|
||||
# Recall = 0 / 1 = 0.0
|
||||
# F1 = 0.0
|
||||
assert result.f1_50 == 0.0
|
||||
assert result.f1_75 == 0.0
|
||||
|
||||
def test_empty_targets(self, detections_50_50):
|
||||
"""Test F1 score with predictions but no targets"""
|
||||
targets = Detections.empty()
|
||||
|
||||
metric = F1Score()
|
||||
result = metric.update(detections_50_50, targets).compute()
|
||||
|
||||
# No targets: TP=0, FP=1, FN=0
|
||||
# Precision = 0 / 1 = 0.0
|
||||
# Recall = 0 / 0 = 0 (by convention)
|
||||
# F1 = 0.0
|
||||
assert result.f1_50 == 0.0
|
||||
assert result.f1_75 == 0.0
|
||||
|
||||
def test_single_class_mixed_results(
|
||||
self, predictions_confidence_ranking, targets_50_50
|
||||
):
|
||||
"""Test F1 score calculation with mixed precision and recall"""
|
||||
metric = F1Score()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# TP=1, FP=1, FN=0
|
||||
# Precision = TP / (TP + FP) = 1 / 2 = 0.5
|
||||
# Recall = TP / (TP + FN) = 1 / 1 = 1.0
|
||||
# F1 = 2 * (0.5 * 1.0) / (0.5 + 1.0) = 1.0 / 1.5 = 2/3 ≈ 0.6667
|
||||
expected_f1 = 2.0 / 3.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
assert_almost_equal(result.f1_75, expected_f1)
|
||||
|
||||
def test_precision_recall_imbalance(
|
||||
self, detections_50_50, targets_two_objects_class_0
|
||||
):
|
||||
"""Test F1 score with different precision and recall scenarios"""
|
||||
metric = F1Score()
|
||||
result = metric.update(detections_50_50, targets_two_objects_class_0).compute()
|
||||
|
||||
# TP=1, FP=0, FN=1
|
||||
# Precision = TP / (TP + FP) = 1 / 1 = 1.0
|
||||
# Recall = TP / (TP + FN) = 1 / 2 = 0.5
|
||||
# F1 = 2 * (1.0 * 0.5) / (1.0 + 0.5) = 1.0 / 1.5 = 2/3 ≈ 0.6667
|
||||
expected_f1 = 2.0 / 3.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
assert_almost_equal(result.f1_75, expected_f1)
|
||||
|
||||
def test_multiple_classes(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test F1 score calculation for multiple classes"""
|
||||
metric = F1Score()
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# Class 0: TP=1, FP=0, FN=0 -> P=1.0, R=1.0, F1=1.0 (weight=1)
|
||||
# Class 1: TP=1, FP=1, FN=0 -> P=0.5, R=1.0, F1=2/3 (weight=1)
|
||||
# Weighted avg: (1*1.0 + 1*2/3) / (1+1) = (1 + 2/3) / 2 = 5/6 ≈ 0.8333
|
||||
expected_f1 = (1.0 + 2.0 / 3.0) / 2.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
assert len(result.matched_classes) == 2
|
||||
assert 0 in result.matched_classes
|
||||
assert 1 in result.matched_classes
|
||||
|
||||
def test_different_iou_thresholds(self, predictions_iou_064, targets_iou_064):
|
||||
"""Test F1 score at different IoU thresholds"""
|
||||
metric = F1Score()
|
||||
result = metric.update(predictions_iou_064, targets_iou_064).compute()
|
||||
|
||||
# IoU = 0.64 > 0.5 but < 0.75
|
||||
# At IoU 0.5: TP=1, FP=0, FN=0 -> P=1.0, R=1.0, F1=1.0
|
||||
# At IoU 0.75: TP=0, FP=1, FN=1 -> P=0.0, R=0.0, F1=0.0
|
||||
assert result.f1_50 == 1.0
|
||||
assert result.f1_75 == 0.0
|
||||
|
||||
def test_confidence_ranking(self, predictions_confidence_ranking, targets_50_50):
|
||||
"""Test that F1 score respects confidence ranking"""
|
||||
metric = F1Score()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# Higher confidence prediction should match the target
|
||||
# TP=1, FP=1, FN=0
|
||||
# Precision = 1/2 = 0.5, Recall = 1/1 = 1.0
|
||||
# F1 = 2 * (0.5 * 1.0) / (0.5 + 1.0) = 2/3
|
||||
expected_f1 = 2.0 / 3.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
|
||||
def test_list_inputs(
|
||||
self, detections_50_50, targets_50_50, prediction_class_1, target_class_1
|
||||
):
|
||||
"""Test F1 score with list inputs"""
|
||||
metric = F1Score()
|
||||
result = metric.update(
|
||||
[detections_50_50, prediction_class_1], [targets_50_50, target_class_1]
|
||||
).compute()
|
||||
|
||||
# Perfect matches for both
|
||||
assert result.f1_50 == 1.0
|
||||
assert result.f1_75 == 1.0
|
||||
|
||||
def test_mismatched_list_lengths(self, detections_50_50, targets_50_50):
|
||||
"""Test that mismatched prediction/target list lengths raise error"""
|
||||
metric = F1Score()
|
||||
|
||||
# Should raise ValueError for mismatched lengths
|
||||
with pytest.raises(ValueError):
|
||||
metric.update([detections_50_50], [targets_50_50, targets_50_50])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"averaging_method",
|
||||
[AveragingMethod.MACRO, AveragingMethod.MICRO, AveragingMethod.WEIGHTED],
|
||||
)
|
||||
def test_averaging_methods(self, averaging_method, detections_50_50, targets_50_50):
|
||||
"""Test different averaging methods"""
|
||||
metric = F1Score(averaging_method=averaging_method)
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give 1.0 regardless of averaging method
|
||||
assert result.f1_50 == 1.0
|
||||
assert result.averaging_method == averaging_method
|
||||
|
||||
def test_macro_averaging(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test MACRO averaging with specific example"""
|
||||
metric = F1Score(averaging_method=AveragingMethod.MACRO)
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# Macro average: (1.0 + 2/3) / 2 = 5/6
|
||||
expected_f1 = (1.0 + 2.0 / 3.0) / 2.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
|
||||
def test_micro_averaging(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test MICRO averaging with specific example"""
|
||||
metric = F1Score(averaging_method=AveragingMethod.MICRO)
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# Micro F1: 4/5 = 0.8
|
||||
expected_f1 = 0.8
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
|
||||
def test_weighted_averaging(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test WEIGHTED averaging with specific example"""
|
||||
metric = F1Score(averaging_method=AveragingMethod.WEIGHTED)
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# Weighted average: 5/6
|
||||
expected_f1 = 5.0 / 6.0
|
||||
assert_almost_equal(result.f1_50, expected_f1)
|
||||
|
|
@ -0,0 +1,296 @@
|
|||
"""
|
||||
Tests for Precision metric
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from supervision.detection.core import Detections
|
||||
from supervision.metrics.core import AveragingMethod, MetricTarget
|
||||
from supervision.metrics.precision import Precision
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detections_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_prediction():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0, matches target
|
||||
[60, 60, 100, 100], # class 0, matches target
|
||||
[200, 200, 240, 240], # class 1, matches target
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.9, 0.8, 0.7]),
|
||||
class_id=np.array([0, 0, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0
|
||||
[60, 60, 100, 100], # class 0
|
||||
[200, 200, 240, 240], # class 1
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 0, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_confidence_ranking():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # low confidence, perfect match
|
||||
[11, 11, 49, 49], # high confidence, good match
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.6, 0.9]), # second has higher confidence
|
||||
class_id=np.array([0, 0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prediction_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
class TestPrecision:
|
||||
def test_initialization_default(self):
|
||||
"""Test that Precision can be initialized with default parameters"""
|
||||
metric = Precision()
|
||||
assert metric._metric_target == MetricTarget.BOXES
|
||||
assert metric.averaging_method == AveragingMethod.WEIGHTED
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_initialization_custom(self):
|
||||
"""Test that Precision can be initialized with custom parameters"""
|
||||
metric = Precision(
|
||||
metric_target=MetricTarget.MASKS,
|
||||
averaging_method=AveragingMethod.MACRO,
|
||||
)
|
||||
assert metric._metric_target == MetricTarget.MASKS
|
||||
assert metric.averaging_method == AveragingMethod.MACRO
|
||||
|
||||
def test_reset(self, dummy_prediction):
|
||||
"""Test that reset() clears all stored data"""
|
||||
metric = Precision()
|
||||
|
||||
# Add some dummy data
|
||||
metric.update(dummy_prediction, dummy_prediction)
|
||||
|
||||
# Verify data was added
|
||||
assert len(metric._predictions_list) == 1
|
||||
assert len(metric._targets_list) == 1
|
||||
|
||||
# Reset and verify lists are empty
|
||||
metric.reset()
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_perfect_match(self, detections_50_50, targets_50_50):
|
||||
"""Test precision with perfect matching predictions and targets"""
|
||||
metric = Precision()
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give precision = 1.0
|
||||
# TP = 1, FP = 0 -> precision = TP / (TP + FP) = 1 / 1 = 1.0
|
||||
# TP = 1, FP = 0 -> precision = TP / (TP + FP) = 1 / 1 = 1.0
|
||||
assert result.precision_at_50 == 1.0
|
||||
assert result.precision_at_75 == 1.0
|
||||
assert len(result.matched_classes) == 1
|
||||
assert result.matched_classes[0] == 0
|
||||
|
||||
def test_no_overlap(self, predictions_no_overlap, targets_no_overlap):
|
||||
"""Test precision with predictions that don't overlap with targets"""
|
||||
metric = Precision()
|
||||
result = metric.update(predictions_no_overlap, targets_no_overlap).compute()
|
||||
|
||||
# No overlap means no TP, only FP
|
||||
# TP = 0, FP = 1 -> precision = TP / (TP + FP) = 0 / 1 = 0.0
|
||||
assert result.precision_at_50 == 0.0
|
||||
assert result.precision_at_75 == 0.0
|
||||
|
||||
def test_empty_predictions(self, targets_50_50):
|
||||
"""Test precision with empty predictions but existing targets"""
|
||||
predictions = Detections.empty()
|
||||
|
||||
metric = Precision()
|
||||
result = metric.update(predictions, targets_50_50).compute()
|
||||
|
||||
# No predictions means TP = 0, FP = 0 -> precision = 0 / 0 = 0
|
||||
assert result.precision_at_50 == 0.0
|
||||
assert result.precision_at_75 == 0.0
|
||||
|
||||
def test_empty_targets(self, detections_50_50):
|
||||
"""Test precision with predictions but no targets"""
|
||||
targets = Detections.empty()
|
||||
|
||||
metric = Precision()
|
||||
result = metric.update(detections_50_50, targets).compute()
|
||||
|
||||
# All predictions are false positives
|
||||
# TP = 0, FP = 1 -> precision = 0 / 1 = 0.0
|
||||
assert result.precision_at_50 == 0.0
|
||||
assert result.precision_at_75 == 0.0
|
||||
|
||||
def test_single_class(self, predictions_confidence_ranking, targets_50_50):
|
||||
"""Test precision calculation for single class with mixed results"""
|
||||
metric = Precision()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# TP = 1 (first prediction), FP = 1 (second prediction)
|
||||
# precision = TP / (TP + FP) = 1 / 2 = 0.5
|
||||
assert result.precision_at_50 == 0.5
|
||||
assert result.precision_at_75 == 0.5
|
||||
|
||||
def test_multiple_classes(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test precision calculation for multiple classes"""
|
||||
metric = Precision()
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# All predictions match targets perfectly
|
||||
# Class 0: TP=2, FP=0 -> precision=1.0 (weight=2)
|
||||
# Class 1: TP=1, FP=0 -> precision=1.0 (weight=1)
|
||||
# Weighted avg: (2*1.0 + 1*1.0) / (2+1) = 3/3 = 1.0
|
||||
assert result.precision_at_50 == 1.0
|
||||
assert result.precision_at_75 == 1.0
|
||||
assert len(result.matched_classes) == 2
|
||||
assert 0 in result.matched_classes
|
||||
assert 1 in result.matched_classes
|
||||
|
||||
def test_different_iou_thresholds(self, predictions_iou_064, targets_iou_064):
|
||||
"""Test precision at different IoU thresholds"""
|
||||
metric = Precision()
|
||||
result = metric.update(predictions_iou_064, targets_iou_064).compute()
|
||||
|
||||
# IoU = 0.64 > 0.5 but < 0.75
|
||||
# Should match at IoU 0.5 but not at 0.75
|
||||
assert result.precision_at_50 == 1.0 # TP=1, FP=0
|
||||
assert result.precision_at_75 == 0.0 # TP=0, FP=1
|
||||
|
||||
def test_confidence_ranking(self, predictions_confidence_ranking, targets_50_50):
|
||||
"""Test that predictions are ranked by confidence"""
|
||||
metric = Precision()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# Higher confidence prediction should match first
|
||||
# TP = 1, FP = 1 -> precision = 0.5
|
||||
assert result.precision_at_50 == 0.5
|
||||
|
||||
def test_list_inputs(
|
||||
self, detections_50_50, targets_50_50, prediction_class_1, target_class_1
|
||||
):
|
||||
"""Test precision with list inputs"""
|
||||
metric = Precision()
|
||||
result = metric.update(
|
||||
[detections_50_50, prediction_class_1], [targets_50_50, target_class_1]
|
||||
).compute()
|
||||
|
||||
# Perfect matches for both
|
||||
assert result.precision_at_50 == 1.0
|
||||
assert result.precision_at_75 == 1.0
|
||||
|
||||
def test_mismatched_list_lengths(self, detections_50_50, targets_50_50):
|
||||
"""Test that mismatched prediction/target list lengths raise error"""
|
||||
metric = Precision()
|
||||
|
||||
# Should raise ValueError for mismatched lengths
|
||||
with pytest.raises(ValueError):
|
||||
metric.update([detections_50_50], [targets_50_50, targets_50_50])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"averaging_method",
|
||||
[AveragingMethod.MACRO, AveragingMethod.MICRO, AveragingMethod.WEIGHTED],
|
||||
)
|
||||
def test_averaging_methods(self, averaging_method, detections_50_50, targets_50_50):
|
||||
"""Test different averaging methods"""
|
||||
metric = Precision(averaging_method=averaging_method)
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give 1.0 regardless of averaging method
|
||||
assert result.precision_at_50 == 1.0
|
||||
assert result.averaging_method == averaging_method
|
||||
|
|
@ -0,0 +1,357 @@
|
|||
"""
|
||||
Tests for Recall metric
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from supervision.detection.core import Detections
|
||||
from supervision.metrics.core import AveragingMethod, MetricTarget
|
||||
from supervision.metrics.recall import Recall
|
||||
from test.helpers import assert_almost_equal
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detections_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_50_50():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 50, 50]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dummy_prediction():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 20, 20]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_no_overlap():
|
||||
return Detections(
|
||||
xyxy=np.array([[100, 100, 110, 110]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_two_objects_class_0():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50],
|
||||
[100, 100, 110, 110],
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0, matches first target
|
||||
[200, 200, 240, 240], # class 1, matches target
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.9, 0.8]),
|
||||
class_id=np.array([0, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_multiple_classes():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0, matched
|
||||
[60, 60, 100, 100], # class 0, missed
|
||||
[200, 200, 240, 240], # class 1, matched
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 0, 1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[15, 15, 55, 55]], dtype=np.float32),
|
||||
confidence=np.array([0.9]),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets_iou_064():
|
||||
return Detections(
|
||||
xyxy=np.array([[10, 10, 60, 60]], dtype=np.float32),
|
||||
class_id=np.array([0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def predictions_confidence_ranking():
|
||||
return Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # low confidence, perfect match
|
||||
[11, 11, 49, 49], # high confidence, good match
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.6, 0.9]), # second has higher confidence
|
||||
class_id=np.array([0, 0]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prediction_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
confidence=np.array([0.8]),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_class_1():
|
||||
return Detections(
|
||||
xyxy=np.array([[60, 60, 100, 100]], dtype=np.float32),
|
||||
class_id=np.array([1]),
|
||||
)
|
||||
|
||||
|
||||
class TestRecall:
|
||||
def test_initialization_default(self):
|
||||
"""Test that Recall can be initialized with default parameters"""
|
||||
metric = Recall()
|
||||
assert metric._metric_target == MetricTarget.BOXES
|
||||
assert metric.averaging_method == AveragingMethod.WEIGHTED
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_initialization_custom(self):
|
||||
"""Test that Recall can be initialized with custom parameters"""
|
||||
metric = Recall(
|
||||
metric_target=MetricTarget.MASKS,
|
||||
averaging_method=AveragingMethod.MACRO,
|
||||
)
|
||||
assert metric._metric_target == MetricTarget.MASKS
|
||||
assert metric.averaging_method == AveragingMethod.MACRO
|
||||
|
||||
def test_reset(self, dummy_prediction):
|
||||
"""Test that reset() clears all stored data"""
|
||||
metric = Recall()
|
||||
|
||||
# Add some dummy data
|
||||
metric.update(dummy_prediction, dummy_prediction)
|
||||
|
||||
# Verify data was added
|
||||
assert len(metric._predictions_list) == 1
|
||||
assert len(metric._targets_list) == 1
|
||||
|
||||
# Reset and verify lists are empty
|
||||
metric.reset()
|
||||
assert metric._predictions_list == []
|
||||
assert metric._targets_list == []
|
||||
|
||||
def test_perfect_match(self, detections_50_50, targets_50_50):
|
||||
"""Test recall with perfect matching predictions and targets"""
|
||||
metric = Recall()
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give recall = 1.0
|
||||
# TP = 1, FN = 0 -> recall = TP / (TP + FN) = 1 / 1 = 1.0
|
||||
assert result.recall_at_50 == 1.0
|
||||
assert result.recall_at_75 == 1.0
|
||||
assert len(result.matched_classes) == 1
|
||||
assert result.matched_classes[0] == 0
|
||||
|
||||
def test_no_overlap(self, predictions_no_overlap, targets_no_overlap):
|
||||
"""Test recall with predictions that don't overlap with targets"""
|
||||
metric = Recall()
|
||||
result = metric.update(predictions_no_overlap, targets_no_overlap).compute()
|
||||
|
||||
# No overlap means no TP, only FN
|
||||
# TP = 0, FN = 1 -> recall = TP / (TP + FN) = 0 / 1 = 0.0
|
||||
assert result.recall_at_50 == 0.0
|
||||
assert result.recall_at_75 == 0.0
|
||||
|
||||
def test_empty_predictions(self, targets_50_50):
|
||||
"""Test recall with empty predictions but existing targets"""
|
||||
predictions = Detections.empty()
|
||||
|
||||
metric = Recall()
|
||||
result = metric.update(predictions, targets_50_50).compute()
|
||||
|
||||
# No predictions means TP = 0, FN = 1 -> recall = 0 / 1 = 0.0
|
||||
assert result.recall_at_50 == 0.0
|
||||
assert result.recall_at_75 == 0.0
|
||||
|
||||
def test_empty_targets(self, detections_50_50):
|
||||
"""Test recall with predictions but no targets"""
|
||||
targets = Detections.empty()
|
||||
|
||||
metric = Recall()
|
||||
result = metric.update(detections_50_50, targets).compute()
|
||||
|
||||
# No targets means TP = 0, FN = 0 -> recall = 0 / 0 = 0
|
||||
assert result.recall_at_50 == 0.0
|
||||
assert result.recall_at_75 == 0.0
|
||||
|
||||
def test_single_class_missed_detections(
|
||||
self, detections_50_50, targets_two_objects_class_0
|
||||
):
|
||||
"""Test recall calculation with some missed detections"""
|
||||
metric = Recall()
|
||||
result = metric.update(detections_50_50, targets_two_objects_class_0).compute()
|
||||
|
||||
# TP = 1 (first target matched), FN = 1 (second target missed)
|
||||
# recall = TP / (TP + FN) = 1 / 2 = 0.5
|
||||
assert_almost_equal(result.recall_at_50, 0.5)
|
||||
assert_almost_equal(result.recall_at_75, 0.5)
|
||||
|
||||
def test_multiple_classes(
|
||||
self, predictions_multiple_classes, targets_multiple_classes
|
||||
):
|
||||
"""Test recall calculation for multiple classes"""
|
||||
metric = Recall()
|
||||
result = metric.update(
|
||||
predictions_multiple_classes, targets_multiple_classes
|
||||
).compute()
|
||||
|
||||
# Class 0: TP=1, FN=1 -> recall=0.5 (weight=2)
|
||||
# Class 1: TP=1, FN=0 -> recall=1.0 (weight=1)
|
||||
# Weighted avg: (2*0.5 + 1*1.0) / (2+1) = 2.0/3 = 0.6667
|
||||
expected_recall = (2 * 0.5 + 1 * 1.0) / (2 + 1)
|
||||
assert_almost_equal(result.recall_at_50, expected_recall)
|
||||
assert_almost_equal(result.recall_at_75, expected_recall)
|
||||
assert len(result.matched_classes) == 2
|
||||
assert 0 in result.matched_classes
|
||||
assert 1 in result.matched_classes
|
||||
|
||||
def test_different_iou_thresholds(self, predictions_iou_064, targets_iou_064):
|
||||
"""Test recall at different IoU thresholds"""
|
||||
metric = Recall()
|
||||
result = metric.update(predictions_iou_064, targets_iou_064).compute()
|
||||
|
||||
# IoU = 0.64 > 0.5 but < 0.75
|
||||
# Should match at IoU 0.5 but not at 0.75
|
||||
assert result.recall_at_50 == 1.0 # TP=1, FN=0
|
||||
assert result.recall_at_75 == 0.0 # TP=0, FN=1
|
||||
|
||||
def test_confidence_ranking(self, predictions_confidence_ranking, targets_50_50):
|
||||
"""Test that higher confidence predictions are preferred for matching"""
|
||||
metric = Recall()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# Target should be matched (by higher confidence prediction)
|
||||
# TP = 1, FN = 0 -> recall = 1.0
|
||||
assert result.recall_at_50 == 1.0
|
||||
|
||||
def test_multiple_predictions_one_target(
|
||||
self, predictions_confidence_ranking, targets_50_50
|
||||
):
|
||||
"""Test recall when multiple predictions compete for one target"""
|
||||
metric = Recall()
|
||||
result = metric.update(predictions_confidence_ranking, targets_50_50).compute()
|
||||
|
||||
# Target should be matched exactly once
|
||||
# TP = 1, FN = 0 -> recall = 1.0
|
||||
assert result.recall_at_50 == 1.0
|
||||
|
||||
def test_list_inputs(
|
||||
self, detections_50_50, targets_50_50, prediction_class_1, target_class_1
|
||||
):
|
||||
"""Test recall with list inputs"""
|
||||
metric = Recall()
|
||||
result = metric.update(
|
||||
[detections_50_50, prediction_class_1], [targets_50_50, target_class_1]
|
||||
).compute()
|
||||
|
||||
# Perfect matches for both
|
||||
assert result.recall_at_50 == 1.0
|
||||
assert result.recall_at_75 == 1.0
|
||||
|
||||
def test_mismatched_list_lengths(self, detections_50_50, targets_50_50):
|
||||
"""Test that mismatched prediction/target list lengths raise error"""
|
||||
metric = Recall()
|
||||
|
||||
# Should raise ValueError for mismatched lengths
|
||||
with pytest.raises(ValueError):
|
||||
metric.update([detections_50_50], [targets_50_50, targets_50_50])
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"averaging_method",
|
||||
[AveragingMethod.MACRO, AveragingMethod.MICRO, AveragingMethod.WEIGHTED],
|
||||
)
|
||||
def test_averaging_methods(self, averaging_method, detections_50_50, targets_50_50):
|
||||
"""Test different averaging methods"""
|
||||
metric = Recall(averaging_method=averaging_method)
|
||||
result = metric.update(detections_50_50, targets_50_50).compute()
|
||||
|
||||
# Perfect match should give 1.0 regardless of averaging method
|
||||
assert result.recall_at_50 == 1.0
|
||||
assert result.averaging_method == averaging_method
|
||||
|
||||
def test_macro_averaging(self):
|
||||
"""Test MACRO averaging with specific example"""
|
||||
# Class 0: 1/2 targets matched -> recall = 0.5
|
||||
# Class 1: 1/1 targets matched -> recall = 1.0
|
||||
# Macro average: (0.5 + 1.0) / 2 = 0.75
|
||||
|
||||
predictions = Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # matches class 0 target 1
|
||||
[200, 200, 240, 240], # matches class 1 target
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
confidence=np.array([0.9, 0.8]),
|
||||
class_id=np.array([0, 1]),
|
||||
)
|
||||
|
||||
targets = Detections(
|
||||
xyxy=np.array(
|
||||
[
|
||||
[10, 10, 50, 50], # class 0, matched
|
||||
[60, 60, 100, 100], # class 0, missed
|
||||
[200, 200, 240, 240], # class 1, matched
|
||||
],
|
||||
dtype=np.float32,
|
||||
),
|
||||
class_id=np.array([0, 0, 1]),
|
||||
)
|
||||
|
||||
metric = Recall(averaging_method=AveragingMethod.MACRO)
|
||||
result = metric.update(predictions, targets).compute()
|
||||
|
||||
# Macro average: (0.5 + 1.0) / 2 = 0.75
|
||||
assert result.recall_at_50 == 0.75
|
||||
Loading…
Reference in New Issue