diff --git a/docs/how_to/benchmark_a_model.md b/docs/how_to/benchmark_a_model.md index e2485a68..16b2b4b6 100644 --- a/docs/how_to/benchmark_a_model.md +++ b/docs/how_to/benchmark_a_model.md @@ -329,6 +329,20 @@ Here, predictions in purple are targets (ground truth), and predictions in teal See [annotator documentation](https://supervision.roboflow.com/latest/detection/annotators/) for even more options. +## Visual Benchmarking + +To inspect where a model succeeds and fails, pass `save_directory_path` to `sv.ConfusionMatrix.benchmark(...)`. For every dataset image it writes a 2x2 result grid — `Ground Truth`, `True Positives`, `False Positives`, and `False Negatives` panels — directly into that directory, reusing the original image filenames. This makes it easy to skim through per-image outcomes alongside the aggregate confusion matrix. + +```python +import supervision as sv + +confusion_matrix = sv.ConfusionMatrix.benchmark( + dataset=test_set, + callback=callback, + save_directory_path="./results", +) +``` + ## Benchmarking Metrics With multiple models, fine details matter. Visual inspection may not be enough. `supervision` provides a collection of metrics that help obtain precise numerical results of model performance. @@ -462,7 +476,7 @@ Yes, if you want to evaluate their bounding boxes. Convert model outputs to `Det ### What is a ConfusionMatrix and how do I use it? -`sv.ConfusionMatrix` visualizes true positives, false positives, and false negatives per class. Create one with `sv.ConfusionMatrix.from_detections(predictions=predictions, targets=targets, classes=classes, conf_threshold=0.5, iou_threshold=0.5)`, then call `metric.plot()` to render a heatmap. +`sv.ConfusionMatrix` visualizes true positives, false positives, and false negatives per class. Create one with `sv.ConfusionMatrix.from_detections(predictions=predictions, targets=targets, classes=classes, conf_threshold=0.5, iou_threshold=0.5)`, then call `confusion_matrix.plot()` to render a heatmap. If you want per-image validation visualizations saved to disk, pass `save_directory_path="./results"` to `sv.ConfusionMatrix.benchmark(...)`; it will write 2x2 result grids directly into that directory using the original image filenames, with `Ground Truth`, `True Positives`, `False Positives`, and `False Negatives` panels. ## Author diff --git a/src/supervision/metrics/detection.py b/src/supervision/metrics/detection.py index 409ac34a..cb76dd3b 100644 --- a/src/supervision/metrics/detection.py +++ b/src/supervision/metrics/detection.py @@ -1,7 +1,10 @@ from __future__ import annotations +import warnings from collections.abc import Callable from dataclasses import dataclass +from pathlib import Path +from typing import cast import matplotlib import matplotlib.pyplot as plt @@ -184,6 +187,409 @@ def _validate_input_tensors( ) +def _split_detections_by_outcome( + predictions: Detections, + targets: Detections, + conf_threshold: float, + iou_threshold: float, + metric_target: MetricTarget = MetricTarget.BOXES, +) -> tuple[Detections, Detections, Detections]: + """ + Split detections into true positives, false positives, and false negatives. + + Matching follows the same attribution logic as + ``ConfusionMatrix.evaluate_detection_batch``: + - matches are computed globally across classes + - same-class matches are prioritized + - higher-IoU matches are preferred + - each prediction and target can be matched at most once + + Cross-class spatial matches are treated as: + - false positives for the prediction + - false negatives for the target + + Args: + predictions: Predicted detections for a single image. + targets: Ground-truth detections for a single image. + conf_threshold: Confidence threshold; predictions below this are excluded. + iou_threshold: IoU threshold; candidate pairs below this are not matched. + metric_target: Coordinate representation to use for IoU computation. + Use ``MetricTarget.ORIENTED_BOUNDING_BOXES`` for rotated-box datasets. + + Returns: + A 3-tuple ``(true_positives, false_positives, false_negatives)`` where + each element is a ``Detections`` instance sliced from the input arrays. + """ + + if predictions.class_id is None: + raise ValueError("Predictions must contain class_id values.") + + if targets.class_id is None: + raise ValueError("Targets must contain class_id values.") + + target_class_ids = targets.class_id + + if predictions.confidence is None: + filtered_predictions = predictions + else: + filtered_predictions = cast( + Detections, + predictions[predictions.confidence >= conf_threshold], + ) + + filtered_prediction_class_ids = filtered_predictions.class_id + if filtered_prediction_class_ids is None: + raise ValueError("Predictions must contain class_id values.") + + prediction_count = len(filtered_predictions) + target_count = len(targets) + + tp_indices: list[int] = [] + fp_indices: list[int] = [] + fn_indices: list[int] = [] + + if prediction_count == 0: + fn_indices = list(range(target_count)) + return ( + cast(Detections, filtered_predictions[tp_indices]), + cast(Detections, filtered_predictions[fp_indices]), + cast(Detections, targets[fn_indices]), + ) + + if target_count == 0: + fp_indices = list(range(prediction_count)) + return ( + cast(Detections, filtered_predictions[tp_indices]), + cast(Detections, filtered_predictions[fp_indices]), + cast(Detections, targets[fn_indices]), + ) + + # IoU computation mirrors evaluate_detection_batch — keep in sync if either changes. + if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + iou_matrix = oriented_box_iou_batch( + boxes_true=np.asarray( + targets.data[ORIENTED_BOX_COORDINATES], dtype=np.float32 + ).reshape(len(targets), 8), + boxes_detection=np.asarray( + filtered_predictions.data[ORIENTED_BOX_COORDINATES], dtype=np.float32 + ).reshape(len(filtered_predictions), 8), + ) + else: + iou_matrix = box_iou_batch( + boxes_true=targets.xyxy, + boxes_detection=filtered_predictions.xyxy, + ) + + target_candidate_indices, prediction_candidate_indices = np.where( + iou_matrix > iou_threshold + ) + + matched_predictions: npt.NDArray[np.bool_] = np.zeros(prediction_count, dtype=bool) + matched_targets: npt.NDArray[np.bool_] = np.zeros(target_count, dtype=bool) + + cross_class_prediction_indices: list[int] = [] + cross_class_target_indices: list[int] = [] + + if len(target_candidate_indices) > 0: + candidate_ious = iou_matrix[ + target_candidate_indices, + prediction_candidate_indices, + ] + + same_class_candidates = ( + target_class_ids[target_candidate_indices] + == filtered_prediction_class_ids[prediction_candidate_indices] + ) + + candidate_order = np.lexsort( + ( + -candidate_ious, + ~same_class_candidates, + ) + ) + + for candidate_index in candidate_order: + target_index = int(target_candidate_indices[candidate_index]) + prediction_index = int(prediction_candidate_indices[candidate_index]) + + if matched_predictions[prediction_index] or matched_targets[target_index]: + continue + + matched_predictions[prediction_index] = True + matched_targets[target_index] = True + + prediction_class = filtered_prediction_class_ids[prediction_index] + target_class = target_class_ids[target_index] + + if prediction_class == target_class: + tp_indices.append(prediction_index) + else: + cross_class_prediction_indices.append(prediction_index) + cross_class_target_indices.append(target_index) + + fp_indices.extend(np.flatnonzero(~matched_predictions).tolist()) + fn_indices.extend(np.flatnonzero(~matched_targets).tolist()) + + fp_indices.extend(cross_class_prediction_indices) + fn_indices.extend(cross_class_target_indices) + + return ( + cast(Detections, filtered_predictions[tp_indices]), + cast(Detections, filtered_predictions[fp_indices]), + cast(Detections, targets[fn_indices]), + ) + + +def _build_error_labels( + detections: Detections, + class_names: list[str] | None, +) -> list[str]: + """Build per-detection label strings for annotation panels. + + Produces labels like ``"cat 0.95"`` (class name + confidence when available) + or numeric class-id strings when ``class_names`` is ``None``. + + Args: + detections: Detections whose labels to build. + class_names: Optional list mapping class integer ids to name strings. + + Returns: + List of label strings, one per detection. Returns empty strings when + ``detections.class_id`` is ``None``. + """ + if detections.class_id is None: + return [""] * len(detections) + + labels: list[str] = [] + for index, class_id in enumerate(detections.class_id): + if class_names is not None and 0 <= int(class_id) < len(class_names): + class_label = class_names[int(class_id)] + else: + class_label = str(int(class_id)) + + confidence = "" + if detections.confidence is not None: + confidence = f" {detections.confidence[index]:.2f}" + + labels.append(f"{class_label}{confidence}") + + return labels + + +def _get_annotation_parameters( + scene: npt.NDArray[np.uint8], +) -> tuple[int, float, int, int, int]: + """Compute adaptive annotation parameters scaled to the panel size. + + Args: + scene: The image panel for which to compute parameters. + + Returns: + A 5-tuple ``(box_thickness, text_scale, text_thickness, text_padding, + font_size)`` where all values are ``int`` except ``text_scale`` (``float``). + """ + height, width = scene.shape[:2] + panel_size = max(min(height, width), 1) + grid_factor = 2 + + font_size = max(18, round(panel_size / (26 * grid_factor))) + box_thickness = max(2, round(font_size / 5)) + text_scale = float(max(1.0, font_size / 20.0)) + text_thickness = max(1, round(font_size / 15.0)) + text_padding = max(6, round(font_size / 3)) + + return box_thickness, text_scale, text_thickness, text_padding, font_size + + +def _annotate_detection_panel( + scene: npt.NDArray[np.uint8], + detections: Detections, + title: str, + class_names: list[str] | None, + annotation_parameters: tuple[int, float, int, int, int], +) -> npt.NDArray[np.uint8]: + """Render detections onto a copy of ``scene`` with a title overlay. + + Args: + scene: Source image panel (not mutated). + detections: Detections to annotate on the panel. + title: Text label rendered in the top-left corner of the panel. + class_names: Optional list mapping class integer ids to name strings. + annotation_parameters: Pre-computed parameters from + ``_get_annotation_parameters``. + + Returns: + Annotated copy of ``scene`` as a ``np.uint8`` array. + """ + import cv2 # lazy: only needed when save_directory_path is set + + from supervision.annotators.core import BoxAnnotator, LabelAnnotator + from supervision.annotators.utils import ColorLookup + from supervision.draw.color import ColorPalette + + panel = scene.copy() + + box_thickness, text_scale, text_thickness, text_padding, font_size = ( + annotation_parameters + ) + + if len(detections) > 0: + box_annotator = BoxAnnotator( + color=ColorPalette.DEFAULT, + color_lookup=ColorLookup.CLASS, + thickness=box_thickness, + ) + label_annotator = LabelAnnotator( + color=ColorPalette.DEFAULT, + color_lookup=ColorLookup.CLASS, + text_scale=text_scale, + text_thickness=text_thickness, + text_padding=text_padding, + ) + labels = _build_error_labels(detections, class_names) + panel = box_annotator.annotate(panel, detections) + panel = label_annotator.annotate(panel, detections, labels=labels) + + title_scale = float(max(1.0, font_size / 18.0)) + title_thickness = max(2, round(font_size / 8)) + panel_height, panel_width = panel.shape[:2] + (title_width, title_height), title_baseline = cv2.getTextSize( + title, + cv2.FONT_HERSHEY_SIMPLEX, + title_scale, + title_thickness, + ) + title_x = max(0, min(text_padding, panel_width - title_width - 1)) + title_y = max(title_height + text_padding, 0) + title_y = min(title_y, max(panel_height - title_baseline - 1, 0)) + + cv2.putText( + panel, + title, + (title_x, title_y), + cv2.FONT_HERSHEY_SIMPLEX, + title_scale, + (240, 240, 240), + title_thickness, + cv2.LINE_AA, + ) + return panel + + +def _save_detection_validation_visualization( + scene: npt.NDArray[np.uint8], + predictions: Detections, + targets: Detections, + save_path: Path, + conf_threshold: float, + iou_threshold: float, + class_names: list[str] | None, + metric_target: MetricTarget = MetricTarget.BOXES, +) -> None: + """Build and save a 2x2 GT/TP/FP/FN mosaic for one image. + + Splits ``predictions`` into true-positive, false-positive, and false-negative + groups using the same matching logic as + ``ConfusionMatrix.evaluate_detection_batch``, renders four annotation panels, + concatenates them into a 2x2 grid, and writes the result to ``save_path``. + + A ``UserWarning`` is emitted if ``cv2.imwrite`` fails (e.g. unsupported + extension or permission error); the benchmark loop continues regardless. + + Args: + scene: The original image for this dataset entry. + predictions: Raw model predictions for ``scene``. + targets: Ground-truth annotations for ``scene``. + save_path: Destination file path for the mosaic image. + conf_threshold: Confidence threshold forwarded to + ``_split_detections_by_outcome``. + iou_threshold: IoU threshold forwarded to ``_split_detections_by_outcome``. + class_names: Optional list mapping class integer ids to name strings. + metric_target: Coordinate representation used for IoU matching. + """ + import cv2 # lazy: only needed when save_directory_path is set + + tp_predictions, fp_predictions, fn_targets = _split_detections_by_outcome( + predictions=predictions, + targets=targets, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + metric_target=metric_target, + ) + + annotation_parameters = _get_annotation_parameters(scene) + + gt_panel = _annotate_detection_panel( + scene=scene, + detections=targets, + title="Ground Truth", + class_names=class_names, + annotation_parameters=annotation_parameters, + ) + tp_panel = _annotate_detection_panel( + scene=scene, + detections=tp_predictions, + title="True Positives", + class_names=class_names, + annotation_parameters=annotation_parameters, + ) + fp_panel = _annotate_detection_panel( + scene=scene, + detections=fp_predictions, + title="False Positives", + class_names=class_names, + annotation_parameters=annotation_parameters, + ) + fn_panel = _annotate_detection_panel( + scene=scene, + detections=fn_targets, + title="False Negatives", + class_names=class_names, + annotation_parameters=annotation_parameters, + ) + + top_row = np.concatenate((gt_panel, tp_panel), axis=1) + bottom_row = np.concatenate((fp_panel, fn_panel), axis=1) + result = np.concatenate((top_row, bottom_row), axis=0) + + panel_height = result.shape[0] // 2 + panel_width = result.shape[1] // 2 + divider_thickness = max(1, min(8, min(panel_height, panel_width) // 32)) + + cv2.rectangle( + result, + (0, 0), + (result.shape[1] - 1, result.shape[0] - 1), + (255, 255, 255), + thickness=divider_thickness, + ) + + center_x = result.shape[1] // 2 + center_y = result.shape[0] // 2 + cv2.line( + result, + (center_x, 0), + (center_x, result.shape[0] - 1), + (255, 255, 255), + divider_thickness, + ) + cv2.line( + result, + (0, center_y), + (result.shape[1] - 1, center_y), + (255, 255, 255), + divider_thickness, + ) + + write_success = cv2.imwrite(str(save_path), result) + if not write_success: + warnings.warn( + f"Failed to write validation image to '{save_path}'.", + UserWarning, + stacklevel=2, + ) + + @deprecated( # type: ignore[untyped-decorator] target=_validate_input_tensors, deprecated_in="0.29.0", @@ -597,6 +1003,8 @@ class ConfusionMatrix: conf_threshold: float = 0.3, iou_threshold: float = 0.5, metric_target: MetricTarget = MetricTarget.BOXES, + *, + save_directory_path: str | Path | None = None, ) -> ConfusionMatrix: """ Calculate confusion matrix from dataset and callback function. @@ -609,6 +1017,10 @@ class ConfusionMatrix: Detections with lower confidence will be excluded. iou_threshold: Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. + save_directory_path: Optional directory where per-image validation + result grids are saved using the original image filenames. Images + are written directly to this directory (no subdirectory is added). + When ``None`` (default), no images are saved. metric_target: The type of detection data to use. Supports `MetricTarget.BOXES` and `MetricTarget.ORIENTED_BOUNDING_BOXES`. Passed through to @@ -643,11 +1055,45 @@ class ConfusionMatrix: # ]) ``` """ + if save_directory_path is not None: + save_directory = Path(save_directory_path) + save_directory.mkdir(parents=True, exist_ok=True) + predictions, targets = [], [] - for _, image, annotation in dataset: + for index, (image_name, image, annotation) in enumerate(dataset): predictions_batch = callback(image) predictions.append(predictions_batch) targets.append(annotation) + + if save_directory_path is not None: + if isinstance(image_name, Path): + image_filename = image_name.name + elif isinstance(image_name, str): + image_filename = Path(image_name).name + else: + image_filename = f"image_{index:06d}.jpg" + + if Path(image_filename).suffix == "": + image_filename = f"{image_filename}.jpg" + + save_path = save_directory / image_filename + if save_path.exists(): + warnings.warn( + f"Validation image '{image_filename}' already exists at " + f"'{save_path}' and will be overwritten.", + UserWarning, + stacklevel=2, + ) + _save_detection_validation_visualization( + scene=image, + predictions=predictions_batch, + targets=annotation, + save_path=save_path, + conf_threshold=conf_threshold, + iou_threshold=iou_threshold, + class_names=dataset.classes, + metric_target=metric_target, + ) return cls.from_detections( predictions=predictions, targets=targets, diff --git a/tests/metrics/test_detection.py b/tests/metrics/test_detection.py index d4f1262c..4b1dd38a 100644 --- a/tests/metrics/test_detection.py +++ b/tests/metrics/test_detection.py @@ -3,6 +3,7 @@ from __future__ import annotations from contextlib import ExitStack as DoesNotRaise from typing import ClassVar +import cv2 import numpy as np import pytest @@ -12,6 +13,7 @@ from supervision.metrics.core import MetricTarget from supervision.metrics.detection import ( ConfusionMatrix, MeanAveragePrecision, + _split_detections_by_outcome, _validate_input_tensors, detections_to_tensor, ) @@ -1210,6 +1212,67 @@ class TestDetectionMetrics: f"wrong-class preds with high IoU might incorrectly match GTs." ) + def test_confusion_matrix_benchmark_saves_validation_visualizations( + self, + tmp_path, + ): + image = np.zeros((32, 32, 3), dtype=np.uint8) + targets = Detections( + xyxy=np.array([[2, 2, 12, 12], [18, 18, 28, 28]], dtype=np.float32), + class_id=np.array([0, 1]), + ) + predictions = Detections( + xyxy=np.array([[2, 2, 12, 12], [4, 18, 12, 28]], dtype=np.float32), + confidence=np.array([0.95, 0.88]), + class_id=np.array([0, 1]), + ) + + class Dataset: + classes: ClassVar[list[str]] = ["cat", "dog"] + + def __iter__(self): + yield "sample.jpg", image, targets + + def callback(_: np.ndarray) -> Detections: + return predictions + + confusion_matrix = ConfusionMatrix.benchmark( + dataset=Dataset(), + callback=callback, + save_directory_path=tmp_path, + ) + + saved_image_path = tmp_path / "sample.jpg" + assert saved_image_path.exists() + + saved_image = cv2.imread(str(saved_image_path)) + assert saved_image is not None + assert saved_image.shape[:2] == (64, 64) + gt_panel = saved_image[:32, :32] + tp_panel = saved_image[:32, 32:] + fp_panel = saved_image[32:, :32] + fn_panel = saved_image[32:, 32:] + + # Assert boxes are rendered in the expected panels, away from borders/dividers. + assert np.any(gt_panel[2:13, 2:13] != 0) + assert np.any(tp_panel[2:13, 2:13] != 0) + # Rows 25-28 are below the panel title text; cols 4-9 are at the left edge + # of the FP box [4,18,12,28] — non-zero only if the box is rendered. + assert np.any(fp_panel[25:29, 4:9] != 0) + # Rows 25-28 are below the panel title text; cols 18-23 are at the left + # edge of the FN target box [18,18,28,28] — non-zero only if box rendered. + assert np.any(fn_panel[25:29, 18:23] != 0) + + # Basic sanity that panels differ. + assert not np.array_equal(gt_panel, tp_panel) + assert not np.array_equal(gt_panel, fp_panel) + assert not np.array_equal(gt_panel, fn_panel) + assert not np.array_equal(tp_panel, fp_panel) + assert not np.array_equal(tp_panel, fn_panel) + assert not np.array_equal(fp_panel, fn_panel) + + assert confusion_matrix.matrix.shape == (3, 3) + @pytest.mark.parametrize( ("predictions", "targets", "metric_target", "exception"), [ @@ -1552,3 +1615,124 @@ class TestDetectionMetrics: metric_target=MetricTarget.BOXES, ) assert cm.metric_target == MetricTarget.BOXES + + +class TestSplitDetectionsByOutcome: + """Tests for _split_detections_by_outcome matching and filtering logic.""" + + def test_confidence_none_all_survive(self): + """Predictions with no confidence score pass threshold regardless.""" + predictions = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + confidence=None, + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + ) + + tp, fp, fn = _split_detections_by_outcome(predictions, targets, 0.5, 0.5) + + assert len(tp) == 1 + assert len(fp) == 0 + assert len(fn) == 0 + + def test_all_below_threshold_returns_zero_tp(self): + """Predictions below conf_threshold are excluded, leaving only FN.""" + predictions = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + confidence=np.array([0.2]), + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + ) + + tp, fp, fn = _split_detections_by_outcome(predictions, targets, 0.5, 0.5) + + assert len(tp) == 0 + assert len(fp) == 0 + assert len(fn) == 1 + + def test_cross_class_spatial_match(self): + """Spatially overlapping but class-mismatched pair → FP + FN.""" + predictions = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + confidence=np.array([0.9]), + class_id=np.array([0]), + ) + targets = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([1]), + ) + + tp, fp, fn = _split_detections_by_outcome(predictions, targets, 0.5, 0.0) + + assert len(tp) == 0 + assert len(fp) == 1 + assert len(fn) == 1 + + def test_empty_predictions_all_fn(self): + """Zero predictions → every target becomes a false negative.""" + predictions = Detections.empty() + predictions.class_id = np.array([], dtype=np.int64) + targets = Detections( + xyxy=np.array([[0, 0, 10, 10], [20, 20, 30, 30]], dtype=np.float32), + class_id=np.array([0, 1]), + ) + + tp, fp, fn = _split_detections_by_outcome(predictions, targets, 0.5, 0.5) + + assert len(tp) == 0 + assert len(fp) == 0 + assert len(fn) == 2 + + def test_empty_targets_all_fp(self): + """Zero targets → every prediction becomes a false positive.""" + predictions = Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + confidence=np.array([0.9]), + class_id=np.array([0]), + ) + targets = Detections.empty() + targets.class_id = np.array([], dtype=np.int64) + + tp, fp, fn = _split_detections_by_outcome(predictions, targets, 0.5, 0.5) + + assert len(tp) == 0 + assert len(fp) == 1 + assert len(fn) == 0 + + @pytest.mark.parametrize( + ("predictions", "targets"), + [ + pytest.param( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=None, + ), + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + ), + id="predictions-class-id-none", + ), + pytest.param( + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=np.array([0]), + ), + Detections( + xyxy=np.array([[0, 0, 10, 10]], dtype=np.float32), + class_id=None, + ), + id="targets-class-id-none", + ), + ], + ) + def test_class_id_none_raises_value_error(self, predictions, targets): + """Missing class_id on either input raises ValueError.""" + with pytest.raises(ValueError, match="class_id"): + _split_detections_by_outcome(predictions, targets, 0.5, 0.5)