xiaoxue_admin
/
supervision
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(metrics): enforce strict typing, sanitize docs & fix numpy 2.0 compat (#2121) 

* fix(pre_commit): 🎨 auto format pre-commit hooks
* refactor(metrics): enforce strict type safety and resolve mypy errors
- Fix incompatible types for Optional arguments in matching functions.
- Add explicit type annotations for empty lists and returns.
- Ensure numpy scalars are cast to native python floats.
- Extract variables to comply with line length limits (E501).
- Resolve 'Missing type parameters' for generic dicts and lists.

---------

Co-authored-by: RobertoHita <roberto.hita@edu.uah.es>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
uahroberto 2026-02-02 17:59:18 +01:00 committed by GitHub
parent 6149bacc77
commit efbb5e2599
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 756 additions and 625 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pyproject.toml
View File

@ -235,13 +235,6 @@ module = [
"supervision.key_points.annotators",
"supervision.key_points.core",
"supervision.key_points.skeletons",
"supervision.metrics.core",
"supervision.metrics.detection",
"supervision.metrics.f1_score",
"supervision.metrics.mean_average_precision",
"supervision.metrics.mean_average_recall",
"supervision.metrics.precision",
"supervision.metrics.recall",
"supervision.metrics.utils.utils",
"supervision.tracker.byte_tracker.core",
"supervision.tracker.byte_tracker.kalman_filter",

4
supervision/metrics/core.py
View File

@ -11,7 +11,7 @@ class Metric(ABC):
"""
@abstractmethod
def update(self, *args, **kwargs) -> Metric:
def update(self, *args: Any, **kwargs: Any) -> Metric:
"""
Add data to the metric, without computing the result.
Return the metric itself to allow method chaining.
@ -26,7 +26,7 @@ class Metric(ABC):
raise NotImplementedError
@abstractmethod
def compute(self, *args, **kwargs) -> Any:
def compute(self, *args: Any, **kwargs: Any) -> Any:
"""
Compute the metric from the internal state and return the result.
"""

164
supervision/metrics/detection.py
View File

@ -18,12 +18,13 @@ def detections_to_tensor(
) -> np.ndarray:
"""
Convert Supervision Detections to numpy tensors for further computation
Args:
detections (sv.Detections): Detections/Targets in the format of sv.Detections
with_confidence (bool): Whether to include confidence in the tensor
detections: Detections/Targets in the format of sv.Detections
with_confidence: Whether to include confidence in the tensor
Returns:
(np.ndarray): Detections as numpy tensors as in (xyxy, class_id,
confidence) order
Detections as numpy tensors as in (xyxy, class_id, confidence) order
"""
if detections.class_id is None:
raise ValueError(
@ -39,10 +40,13 @@ def detections_to_tensor(
)
arrays_to_concat.append(np.expand_dims(detections.confidence, 1))
return np.concatenate(arrays_to_concat, axis=1)
result: np.ndarray = np.concatenate(arrays_to_concat, axis=1)
return result
def validate_input_tensors(predictions: list[np.ndarray], targets: list[np.ndarray]):
def validate_input_tensors(
predictions: list[np.ndarray], targets: list[np.ndarray]
) -> None:
"""
Checks for shape consistency of input tensors.
"""
@ -76,13 +80,12 @@ class ConfusionMatrix:
Confusion matrix for object detection tasks.
Attributes:
matrix (np.ndarray): An 2D `np.ndarray` of shape
`(len(classes) + 1, len(classes) + 1)`
matrix: An 2D `np.ndarray` of shape `(len(classes) + 1, len(classes) + 1)`
containing the number of `TP`, `FP`, `FN` and `TN` for each class.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`.
classes: Model class names.
conf_threshold: Detection confidence threshold between `0` and `1`.
Detections with lower confidence will be excluded from the matrix.
iou_threshold (float): Detection IoU threshold between `0` and `1`.
iou_threshold: Detection IoU threshold between `0` and `1`.
Detections with lower IoU will be classified as `FP`.
"""
@ -104,16 +107,16 @@ class ConfusionMatrix:
Calculate confusion matrix based on predicted and ground-truth detections.
Args:
targets (List[Detections]): Detections objects from ground-truth.
predictions (List[Detections]): Detections objects predicted by the model.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`.
targets: Detections objects from ground-truth.
predictions: Detections objects predicted by the model.
classes: Model class names.
conf_threshold: Detection confidence threshold between `0` and `1`.
Detections with lower confidence will be excluded.
iou_threshold (float): Detection IoU threshold between `0` and `1`.
iou_threshold: Detection IoU threshold between `0` and `1`.
Detections with lower IoU will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
New instance of ConfusionMatrix.
Examples:
>>> import numpy as np
@ -169,22 +172,22 @@ class ConfusionMatrix:
Calculate confusion matrix based on predicted and ground-truth detections.
Args:
predictions (List[np.ndarray]): Each element of the list describes a single
predictions: Each element of the list describes a single
image and has `shape = (M, 6)` where `M` is the number of detected
objects. Each row is expected to be in
`(x_min, y_min, x_max, y_max, class, conf)` format.
targets (List[np.ndarray]): Each element of the list describes a single
targets: Each element of the list describes a single
image and has `shape = (N, 5)` where `N` is the number of
ground-truth objects. Each row is expected to be in
`(x_min, y_min, x_max, y_max, class)` format.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`.
classes: Model class names.
conf_threshold: Detection confidence threshold between `0` and `1`.
Detections with lower confidence will be excluded.
iou_threshold (float): Detection iou threshold between `0` and `1`.
iou_threshold: Detection iou threshold between `0` and `1`.
Detections with lower iou will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
New instance of ConfusionMatrix.
Examples:
>>> import supervision as sv
@ -244,22 +247,22 @@ class ConfusionMatrix:
Calculate confusion matrix for a batch of detections for a single image.
Args:
predictions (np.ndarray): Batch prediction. Describes a single image and
predictions: Batch prediction. Describes a single image and
has `shape = (M, 6)` where `M` is the number of detected objects.
Each row is expected to be in
`(x_min, y_min, x_max, y_max, class, conf)` format.
targets (np.ndarray): Batch target labels. Describes a single image and
targets: Batch target labels. Describes a single image and
has `shape = (N, 5)` where `N` is the number of ground-truth objects.
Each row is expected to be in
`(x_min, y_min, x_max, y_max, class)` format.
num_classes (int): Number of classes.
conf_threshold (float): Detection confidence threshold between `0` and `1`.
num_classes: Number of classes.
conf_threshold: Detection confidence threshold between `0` and `1`.
Detections with lower confidence will be excluded.
iou_threshold (float): Detection iou threshold between `0` and `1`.
iou_threshold: Detection iou threshold between `0` and `1`.
Detections with lower iou will be classified as `FP`.
Returns:
np.ndarray: Confusion matrix based on a single image.
Confusion matrix based on a single image.
"""
result_matrix = np.zeros((num_classes + 1, num_classes + 1))
@ -304,8 +307,8 @@ class ConfusionMatrix:
for i, detection_class_value in enumerate(detection_classes):
if not any(matched_detection_idx == i):
result_matrix[num_classes, detection_class_value] += 1 # FP
return result_matrix
final_result_matrix: np.ndarray = result_matrix
return final_result_matrix
@staticmethod
def _drop_extra_matches(matches: np.ndarray) -> np.ndarray:
@ -332,16 +335,16 @@ class ConfusionMatrix:
Calculate confusion matrix from dataset and callback function.
Args:
dataset (DetectionDataset): Object detection dataset used for evaluation.
callback (Callable[[np.ndarray], Detections]): Function that takes an image
as input and returns Detections object.
conf_threshold (float): Detection confidence threshold between `0` and `1`.
dataset: Object detection dataset used for evaluation.
callback: Function that takes an image as input and returns a
Detections object.
conf_threshold: Detection confidence threshold between `0` and `1`.
Detections with lower confidence will be excluded.
iou_threshold (float): Detection IoU threshold between `0` and `1`.
iou_threshold: Detection IoU threshold between `0` and `1`.
Detections with lower IoU will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
New instance of ConfusionMatrix.
Example:
```python
@ -394,16 +397,16 @@ class ConfusionMatrix:
Create confusion matrix plot and save it at selected location.
Args:
save_path (Optional[str]): Path to save the plot. If not provided,
save_path: Path to save the plot. If not provided,
plot will be displayed.
title (Optional[str]): Title of the plot.
classes (Optional[List[str]]): List of classes to be displayed on the plot.
title: Title of the plot.
classes: List of classes to be displayed on the plot.
If not provided, all classes will be displayed.
normalize (bool): If True, normalize the confusion matrix.
fig_size (Tuple[int, int]): Size of the plot.
normalize: If True, normalize the confusion matrix.
fig_size: Size of the plot.
Returns:
matplotlib.figure.Figure: Confusion matrix plot.
Confusion matrix plot.
"""
array = self.matrix.copy()
@ -493,13 +496,13 @@ class MeanAveragePrecision:
Mean Average Precision for object detection tasks.
Attributes:
map50_95 (float): Mean Average Precision (mAP) calculated over IoU thresholds
map50_95: Mean Average Precision (mAP) calculated over IoU thresholds
ranging from `0.50` to `0.95` with a step size of `0.05`.
map50 (float): Mean Average Precision (mAP) calculated specifically at
map50: Mean Average Precision (mAP) calculated specifically at
an IoU threshold of `0.50`.
map75 (float): Mean Average Precision (mAP) calculated specifically at
map75: Mean Average Precision (mAP) calculated specifically at
an IoU threshold of `0.75`.
per_class_ap50_95 (np.ndarray): Average Precision (AP) values calculated over
per_class_ap50_95: Average Precision (AP) values calculated over
IoU thresholds ranging from `0.50` to `0.95` with a step size of `0.05`,
provided for each individual class.
"""
@ -519,10 +522,10 @@ class MeanAveragePrecision:
Calculate mean average precision based on predicted and ground-truth detections.
Args:
targets (List[Detections]): Detections objects from ground-truth.
predictions (List[Detections]): Detections objects predicted by the model.
targets: Detections objects from ground-truth.
predictions: Detections objects predicted by the model.
Returns:
MeanAveragePrecision: New instance of ConfusionMatrix.
New instance of ConfusionMatrix.
Examples:
>>> import numpy as np
@ -569,11 +572,11 @@ class MeanAveragePrecision:
Calculate mean average precision from dataset and callback function.
Args:
dataset (DetectionDataset): Object detection dataset used for evaluation.
callback (Callable[[np.ndarray], Detections]): Function that takes
dataset: Object detection dataset used for evaluation.
callback: Function that takes
an image as input and returns Detections object.
Returns:
MeanAveragePrecision: New instance of MeanAveragePrecision.
New instance of MeanAveragePrecision.
Example:
```python
@ -617,16 +620,16 @@ class MeanAveragePrecision:
detections at different threshold.
Args:
predictions (List[np.ndarray]): Each element of the list describes
predictions: Each element of the list describes
a single image and has `shape = (M, 6)` where `M` is
the number of detected objects. Each row is expected to be
in `(x_min, y_min, x_max, y_max, class, conf)` format.
targets (List[np.ndarray]): Each element of the list describes a single
targets: Each element of the list describes a single
image and has `shape = (N, 5)` where `N` is the
number of ground-truth objects. Each row is expected to be in
`(x_min, y_min, x_max, y_max, class)` format.
Returns:
MeanAveragePrecision: New instance of MeanAveragePrecision.
New instance of MeanAveragePrecision.
Examples:
>>> import supervision as sv
@ -691,7 +694,7 @@ class MeanAveragePrecision:
map50_95 = average_precisions.mean()
else:
map50, map75, map50_95 = 0, 0, 0
average_precisions = []
average_precisions = np.array([])
return cls(
map50_95=map50_95,
@ -707,11 +710,11 @@ class MeanAveragePrecision:
the recall and precision curves.
Args:
recall (np.ndarray): The recall curve.
precision (np.ndarray): The precision curve.
recall: The recall curve.
precision: The precision curve.
Returns:
float: Average precision.
Average precision.
"""
extended_recall = np.concatenate(([0.0], recall, [1.0]))
extended_precision = np.concatenate(([1.0], precision, [0.0]))
@ -722,8 +725,18 @@ class MeanAveragePrecision:
interpolated_precision = np.interp(
interpolated_recall_levels, extended_recall, max_accumulated_precision
)
average_precision = np.trapz(interpolated_precision, interpolated_recall_levels)
return average_precision
# Check if we are running on NumPy 2.0+ or older
if hasattr(np, "trapezoid"):
average_precision = np.trapezoid(
interpolated_precision, interpolated_recall_levels
)
else:
average_precision = np.trapz( # type: ignore[attr-defined]
interpolated_precision, interpolated_recall_levels
)
return float(average_precision)
@staticmethod
def _match_detection_batch(
@ -733,18 +746,18 @@ class MeanAveragePrecision:
Match predictions with target labels based on IoU levels.
Args:
predictions (np.ndarray): Batch prediction. Describes a single image and
predictions: Batch prediction. Describes a single image and
has `shape = (M, 6)` where `M` is the number of detected objects.
Each row is expected to be in
`(x_min, y_min, x_max, y_max, class, conf)` format.
targets (np.ndarray): Batch target labels. Describes a single image and
targets: Batch target labels. Describes a single image and
has `shape = (N, 5)` where `N` is the number of ground-truth objects.
Each row is expected to be in
`(x_min, y_min, x_max, y_max, class)` format.
iou_thresholds (np.ndarray): Array contains different IoU thresholds.
iou_thresholds: Array contains different IoU thresholds.
Returns:
np.ndarray: Matched prediction with target labels result.
Matched prediction with target labels result.
"""
num_predictions, num_iou_levels = predictions.shape[0], iou_thresholds.shape[0]
correct = np.zeros((num_predictions, num_iou_levels), dtype=bool)
@ -765,8 +778,8 @@ class MeanAveragePrecision:
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
correct[matches[:, 1].astype(int), i] = True
return correct
result: np.ndarray = correct
return result
@staticmethod
def _average_precisions_per_class(
@ -781,14 +794,14 @@ class MeanAveragePrecision:
Source: https://github.com/rafaelpadilla/Object-Detection-Metrics.
Args:
matches (np.ndarray): True positives.
prediction_confidence (np.ndarray): Objectness value from 0-1.
prediction_class_ids (np.ndarray): Predicted object classes.
true_class_ids (np.ndarray): True object classes.
eps (float): Small value to prevent division by zero.
matches: True positives.
prediction_confidence: Objectness value from 0-1.
prediction_class_ids: Predicted object classes.
true_class_ids: True object classes.
eps: Small value to prevent division by zero.
Returns:
np.ndarray: Average precision for different IoU levels.
Average precision for different IoU levels.
"""
sorted_indices = np.argsort(-prediction_confidence)
matches = matches[sorted_indices]
@ -819,4 +832,5 @@ class MeanAveragePrecision:
)
)
return average_precisions
result: np.ndarray = average_precisions
return result

92
supervision/metrics/f1_score.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import numpy as np
from matplotlib import pyplot as plt
@ -68,8 +68,8 @@ class F1Score(Metric):
Initialize the F1Score metric.
Args:
metric_target (MetricTarget): The type of detection data to use.
averaging_method (AveragingMethod): The averaging method used to compute the
metric_target: The type of detection data to use.
averaging_method: The averaging method used to compute the
F1 scores. Determines how the F1 scores are aggregated across classes.
"""
self._metric_target = metric_target
@ -94,11 +94,11 @@ class F1Score(Metric):
Add new predictions and targets to the metric, but do not compute the result.
Args:
predictions (Union[Detections, List[Detections]]): The predicted detections.
targets (Union[Detections, List[Detections]]): The target detections.
predictions: The predicted detections.
targets: The target detections.
Returns:
(F1Score): The updated metric instance.
The updated metric instance.
"""
if not isinstance(predictions, list):
predictions = [predictions]
@ -122,7 +122,7 @@ class F1Score(Metric):
data, at different IoU thresholds.
Returns:
(F1ScoreResult): The F1 score metric result.
The F1 score metric result.
"""
result = self._compute(self._predictions_list, self._targets_list)
@ -149,7 +149,7 @@ class F1Score(Metric):
self, predictions_list: list[Detections], targets_list: list[Detections]
) -> F1ScoreResult:
iou_thresholds = np.linspace(0.5, 0.95, 10)
stats = []
stats: list[Any] = []
for predictions, targets in zip(predictions_list, targets_list):
prediction_contents = self._detections_content(predictions)
@ -181,7 +181,14 @@ class F1Score(Metric):
)
matches = self._match_detection_batch(
predictions.class_id, targets.class_id, iou, iou_thresholds
predictions.class_id
if predictions.class_id is not None
else np.array([]),
targets.class_id
if targets.class_id is not None
else np.array([]),
iou,
iou_thresholds,
)
stats.append(
(
@ -283,7 +290,8 @@ class F1Score(Metric):
correct[matches[:, 1].astype(int), i] = True
return correct
result_correct: np.ndarray = correct
return result_correct
@staticmethod
def _compute_confusion_matrix(
@ -298,18 +306,18 @@ class F1Score(Metric):
Assumes the matches and prediction_class_ids are sorted by confidence
in descending order.
Arguments:
sorted_matches: np.ndarray, bool, shape (P, Th), that is True
Args:
sorted_matches: shape (P, Th), that is True
if the prediction is a true positive at the given IoU threshold.
sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
sorted_prediction_class_ids: shape (P,), containing
the class id for each prediction.
unique_classes: np.ndarray, int, shape (C,), containing the unique
unique_classes: shape (C,), containing the unique
class ids.
class_counts: np.ndarray, int, shape (C,), containing the number
class_counts: shape (C,), containing the number
of true instances for each class.
Returns:
np.ndarray, shape (C, Th, 3), containing the true positives, false
shape (C, Th, 3), containing the true positives, false
positives, and false negatives for each class and IoU threshold.
"""
@ -338,7 +346,8 @@ class F1Score(Metric):
[true_positives, false_positives, false_negatives], axis=1
)
return confusion_matrix
result_confusion_matrix: np.ndarray = confusion_matrix
return result_confusion_matrix
@staticmethod
def _compute_f1(confusion_matrix: np.ndarray) -> np.ndarray:
@ -346,11 +355,11 @@ class F1Score(Metric):
Broadcastable function, computing the F1 score from the confusion matrix.
Arguments:
confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
confusion_matrix: shape (N, ..., 3), where the last dimension
contains the true positives, false positives, and false negatives.
Returns:
np.ndarray, shape (N, ...), containing the F1 score for each element.
shape (N, ...), containing the F1 score for each element.
"""
if not confusion_matrix.shape[-1] == 3:
raise ValueError(
@ -365,7 +374,8 @@ class F1Score(Metric):
denominator = 2 * true_positives + false_positives + false_negatives
f1_score = np.where(denominator == 0, 0, 2 * true_positives / denominator)
return f1_score
result_f1_score: np.ndarray = f1_score
return result_f1_score
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@ -380,17 +390,21 @@ class F1Score(Metric):
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
obb = detections.data.get(ORIENTED_BOX_COORDINATES)
if obb is not None and len(obb) > 0:
return np.array(obb, dtype=np.float32)
result_obb: np.ndarray = np.array(obb, dtype=np.float32)
return result_obb
return self._make_empty_content()
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _make_empty_content(self) -> np.ndarray:
if self._metric_target == MetricTarget.BOXES:
return np.empty((0, 4), dtype=np.float32)
empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32)
return empty_boxes
if self._metric_target == MetricTarget.MASKS:
return np.empty((0, 0, 0), dtype=bool)
empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool)
return empty_masks
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
return np.empty((0, 4, 2), dtype=np.float32)
empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32)
return empty_obb
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _filter_detections_by_size(
@ -448,24 +462,24 @@ class F1ScoreResult:
Defaults to `0` if no detections or targets were provided.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
metric_target: the type of data used for the metric -
boxes, masks or oriented bounding boxes.
averaging_method (AveragingMethod): the averaging method used to compute the
averaging_method: the averaging method used to compute the
F1 scores. Determines how the F1 scores are aggregated across classes.
f1_50 (float): the F1 score at IoU threshold of `0.5`.
f1_75 (float): the F1 score at IoU threshold of `0.75`.
f1_scores (np.ndarray): the F1 scores at each IoU threshold.
f1_50: the F1 score at IoU threshold of `0.5`.
f1_75: the F1 score at IoU threshold of `0.75`.
f1_scores: the F1 scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
f1_per_class (np.ndarray): the F1 scores per class and IoU threshold.
f1_per_class: the F1 scores per class and IoU threshold.
Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
iou_thresholds: the IoU thresholds used in the calculations.
matched_classes: the class IDs of all matched classes.
Corresponds to the rows of `f1_per_class`.
small_objects (Optional[F1ScoreResult]): the F1 metric results
small_objects: the F1 metric results
for small objects (area < 32²).
medium_objects (Optional[F1ScoreResult]): the F1 metric results
medium_objects: the F1 metric results
for medium objects (32² ≤ area < 96²).
large_objects (Optional[F1ScoreResult]): the F1 metric results
large_objects: the F1 metric results
for large objects (area ≥ 96²).
"""
@ -474,11 +488,11 @@ class F1ScoreResult:
@property
def f1_50(self) -> float:
return self.f1_scores[0]
return float(self.f1_scores[0])
@property
def f1_75(self) -> float:
return self.f1_scores[5]
return float(self.f1_scores[5])
f1_scores: np.ndarray
f1_per_class: np.ndarray
@ -544,7 +558,7 @@ class F1ScoreResult:
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
@ -569,7 +583,7 @@ class F1ScoreResult:
return pd.DataFrame(pandas_data, index=[0])
def plot(self):
def plot(self) -> None:
"""
Plot the F1 results.

400
supervision/metrics/mean_average_precision.py
View File

@ -12,8 +12,8 @@ from typing import TYPE_CHECKING, Any
import numpy as np
from matplotlib import pyplot as plt
from supervision import box_iou_batch_with_jaccard
from supervision.detection.core import Detections
from supervision.detection.utils.iou_and_nms import box_iou_batch_with_jaccard
from supervision.draw.color import LEGACY_COLOR_PALETTE
from supervision.metrics.core import Metric, MetricTarget
from supervision.metrics.utils.utils import ensure_pandas_installed
@ -30,25 +30,22 @@ class MeanAveragePrecisionResult:
Defaults to `0` when no detections or targets are present.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
metric_target: the type of data used for the metric -
boxes, masks or oriented bounding boxes.
class_agnostic (bool): When computing class-agnostic results, class ID
is_class_agnostic: When computing class-agnostic results, class ID
is set to `-1`.
mAP_map50_95 (float): the mAP score at IoU thresholds from `0.5` to `0.95`.
mAP_map50 (float): the mAP score at IoU threshold of `0.5`.
mAP_map75 (float): the mAP score at IoU threshold of `0.75`.
mAP_scores (np.ndarray): the mAP scores at each IoU threshold.
mAP_scores: the mAP scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
ap_per_class (np.ndarray): the average precision scores per
ap_per_class: the average precision scores per
class and IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
iou_thresholds: the IoU thresholds used in the calculations.
matched_classes: the class IDs of all matched classes.
Corresponds to the rows of `ap_per_class`.
small_objects (Optional[MeanAveragePrecisionResult]): the mAP results
small_objects: the mAP results
for small objects (area < 32²).
medium_objects (Optional[MeanAveragePrecisionResult]): the mAP results
medium_objects: the mAP results
for medium objects (32² ≤ area < 96²).
large_objects (Optional[MeanAveragePrecisionResult]): the mAP results
large_objects: the mAP results
for large objects (area ≥ 96²).
"""
@ -57,19 +54,22 @@ class MeanAveragePrecisionResult:
@property
def map50_95(self) -> float:
"""the mAP score at IoU thresholds from `0.5` to `0.95`."""
valid_scores = self.mAP_scores[self.mAP_scores > -1]
if len(valid_scores) > 0:
return valid_scores.mean()
return float(valid_scores.mean())
else:
return -1
@property
def map50(self) -> float:
return self.mAP_scores[0]
"""the mAP score at IoU threshold of `0.5`."""
return float(self.mAP_scores[0])
@property
def map75(self) -> float:
return self.mAP_scores[5]
"""the mAP score at IoU threshold of `0.75`."""
return float(self.mAP_scores[5])
mAP_scores: np.ndarray
ap_per_class: np.ndarray
@ -95,6 +95,20 @@ class MeanAveragePrecisionResult:
Average Precision (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.629
```
"""
if (
self.small_objects is None
or self.medium_objects is None
or self.large_objects is None
):
return (
f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | "
f"maxDets=100 ] = {self.map50_95:.3f}\n"
f"Average Precision (AP) @[ IoU=0.50 | area= all | "
f"maxDets=100 ] = {self.map50:.3f}\n"
f"Average Precision (AP) @[ IoU=0.75 | area= all | "
f"maxDets=100 ] = {self.map75:.3f}"
)
return (
f"Average Precision (AP) @[ IoU=0.50:0.95 | area= all | "
f"maxDets=100 ] = {self.map50_95:.3f}\n"
@ -115,7 +129,7 @@ class MeanAveragePrecisionResult:
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
@ -145,7 +159,7 @@ class MeanAveragePrecisionResult:
index=[0],
)
def plot(self):
def plot(self) -> None:
"""
Plot the mAP results.
@ -224,13 +238,19 @@ class EvaluationDataset:
"""
Constructor of EvaluationDataset object used to evaluate models with
Mean Average Precision.
Args:
targets (dict): The targets (ground truth) of the dataset in a the
COCO format.
targets: The targets (ground truth) of the dataset in a the
COCO format.
"""
# Initialize members
self.dataset, self.anns, self.cats, self.imgs = dict(), dict(), dict(), dict()
self.img_to_anns, self.cat_to_imgs = defaultdict(list), defaultdict(list)
# Initialize members
self.dataset: dict[str, Any] = dict()
self.anns: dict[int, Any] = dict()
self.cats: dict[int, Any] = dict()
self.imgs: dict[int, Any] = dict()
self.img_to_anns: dict[int, list[Any]] = defaultdict(list)
self.cat_to_imgs: dict[int, list[int]] = defaultdict(list)
if targets is None:
return
@ -240,10 +260,10 @@ class EvaluationDataset:
self.create_class_members()
@classmethod
def empty(cls):
def empty(cls) -> EvaluationDataset:
return cls(targets=None)
def create_class_members(self):
def create_class_members(self) -> None:
"""
Create index elements for the dataset.
"""
@ -275,25 +295,26 @@ class EvaluationDataset:
def get_annotation_ids(
self,
img_ids: list[int] = [],
cat_ids: list[int] = [],
area_range: tuple[float, float] = [],
img_ids: list[int] | None = None,
cat_ids: list[int] | None = None,
area_range: tuple[float, float] | None = None,
iscrowd: bool = False,
):
) -> list[int]:
"""
Get annotation ids that satisfy given filter conditions.
Args:
img_ids (list): ids of the images that we want to retrieve.
cat_ids (list): ids of the categories that we want to retrieve.
area_range (tuple): area range of the annotations that we want to retrieve
in the format [min_area, max_area].
iscrowd (bool): if annotations to retrieve are `iscrowded=1`.
img_ids: ids of the images that we want to retrieve.
cat_ids: ids of the categories that we want to retrieve.
area_range: area range of the annotations that we want to retrieve
in the format [min_area, max_area].
iscrowd: if annotations to retrieve are `iscrowded=1`.
"""
# If there are no filters, we use all annotations
if len(img_ids) == len(cat_ids) == len(area_range) == 0:
if not img_ids and not cat_ids and not area_range:
anns = self.dataset["annotations"]
else:
if len(img_ids) != 0:
if img_ids:
lists = [
self.img_to_anns[img_id]
for img_id in img_ids
@ -306,18 +327,16 @@ class EvaluationDataset:
# Filter by category
anns = (
anns
if len(cat_ids) == 0
if not cat_ids
else [ann for ann in anns if ann["category_id"] in cat_ids]
)
# Filter by area
anns = (
anns
if len(area_range) == 0
if not area_range
else [
ann
for ann in anns
if ann["area"] > area_range[0] and ann["area"] < area_range[1]
ann for ann in anns if area_range[0] < ann["area"] < area_range[1]
]
)
@ -330,21 +349,23 @@ class EvaluationDataset:
def get_category_ids(
self,
cat_names: list[str] = [],
supercategory_names: list[str] = [],
cat_ids: list[int] = [],
cat_names: list[str] | None = None,
supercategory_names: list[str] | None = None,
cat_ids: list[int] | None = None,
) -> list[int]:
"""
Get category ids that satisfy given filter conditions.
Args:
cat_names (list): names of the categories to retrieve.
supercategory_names (list): names of the supercategories to retrieve.
cat_ids (list): ids of the categories to retrieve.
cat_names: names of the categories to retrieve.
supercategory_names: names of the supercategories to retrieve.
cat_ids: ids of the categories to retrieve.
Returns:
ids (list): integer array of category ids.
ids: integer array of category ids.
"""
# If there are no filters, we use all categories
if len(cat_names) == len(supercategory_names) == len(cat_ids) == 0:
if not cat_names and not supercategory_names and not cat_ids:
cats = self.dataset["categories"]
else:
cats = self.dataset["categories"]
@ -352,14 +373,14 @@ class EvaluationDataset:
# Filter by name
cats = (
cats
if len(cat_names) == 0
if not cat_names
else [cat for cat in cats if cat["name"] in cat_names]
)
# Filter by supercategory
cats = (
cats
if len(supercategory_names) == 0
if not supercategory_names
else [
cat for cat in cats if cat["supercategory"] in supercategory_names
]
@ -367,54 +388,63 @@ class EvaluationDataset:
# Filter by id
cats = (
cats
if len(cat_ids) == 0
else [cat for cat in cats if cat["id"] in cat_ids]
cats if not cat_ids else [cat for cat in cats if cat["id"] in cat_ids]
)
ids = [cat["id"] for cat in cats]
return ids
def get_image_ids(
self,
img_ids: list[int] = [],
cat_ids: list[int] = [],
img_ids: list[int] | None = None,
cat_ids: list[int] | None = None,
) -> list[int]:
"""
Get image ids that satisfy given filter conditions.
Args:
img_ids (list): ids of the images to retrieve.
cat_ids (list): ids of the categories to retrieve.
img_ids: ids of the images to retrieve.
cat_ids: ids of the categories to retrieve.
Returns:
ids (list): integer array of image ids.
ids: integer array of image ids.
"""
# If there are no filters, we use all images
if len(img_ids) == len(cat_ids) == 0:
if not img_ids and not cat_ids:
ids = self.imgs.keys()
return list(ids)
ids = set(img_ids)
for i, cat_id in enumerate(cat_ids):
if i == 0 and len(ids) == 0:
ids = set(self.cat_to_imgs[cat_id])
else:
ids &= set(self.cat_to_imgs[cat_id])
return list(ids)
ids_set = set(img_ids) if img_ids else set()
def get_annotations(self, ids: list[int] = []) -> list[dict]:
if cat_ids:
for i, cat_id in enumerate(cat_ids):
if i == 0 and not ids_set:
ids_set = set(self.cat_to_imgs[cat_id])
else:
ids_set &= set(self.cat_to_imgs[cat_id])
return list(ids_set)
def get_annotations(self, ids: list[int] | None = None) -> list[dict[str, Any]]:
"""
Get annotations with the specified ids.
Args:
ids (list): integer ids specifying annotations.
ids: integer ids specifying annotations.
Returns:
anns (list): loaded annotations.
anns: loaded annotations.
"""
if ids is None:
return []
return [self.anns[idx] for idx in ids]
def load_predictions(self, predictions: list[dict]) -> EvaluationDataset:
def load_predictions(self, predictions: list[dict[str, Any]]) -> EvaluationDataset:
"""
Load prediction result into an EvaluationDataset object.
Args:
predictions (list): prediction result.
predictions: prediction result.
Returns:
EvaluationDataset object representing the predictions.
"""
@ -464,7 +494,9 @@ class EvaluationDataset:
# Make segmentation from bounding box coordinates
if "segmentation" not in pred:
pred["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
pred["area"] = w * h
# Use provided area if available
if "area" not in pred:
pred["area"] = w * h
pred["id"] = idx + 1
# For predictions we set iscrowd to 0
pred["iscrowd"] = 0
@ -498,10 +530,12 @@ class COCOEvaluatorParameters:
Parameters for COCOEvaluator
"""
def __init__(self):
def __init__(self) -> None:
"""Initialize all parameters for evaluation"""
self.img_ids, self.cat_ids = [], []
self.img_ids: list[int] = []
self.cat_ids: list[int] = []
# IoU thresholds [0.5, 0.55, 0.6, 0.65, ..., 0.95]
self.iou_thrs = np.linspace(
0.5, 0.95, int(np.round((0.95 - 0.5) / 0.05)) + 1, endpoint=True
@ -513,7 +547,7 @@ class COCOEvaluatorParameters:
# 3 maximum detection thresholds [1, 10, 100]
self.max_dets = [1, 10, 100]
# Area ranges [0, 1e5], [0, 32], [32, 96], [96, 1e5]
self.area_range = [
self.area_range: list[list[float]] = [
[0, MAX_ALL_OBJECT_AREA],
[0, SMALL_OBJECT_AREA],
[SMALL_OBJECT_AREA, MEDIUM_OBJECT_AREA],
@ -533,8 +567,8 @@ class COCOEvaluator:
Constructor of COCOEvaluator object.
Args:
coco_targets (EvaluationDataset): The dataset with the ground truths.
coco_predictions (EvaluationDataset): The dataset with the predictions.
coco_targets: The dataset with the ground truths.
coco_predictions: The dataset with the predictions.
"""
if coco_targets is None:
raise ValueError("coco_targets must be provided")
@ -546,23 +580,23 @@ class COCOEvaluator:
# List of dictionaries containing the evaluation results
# len(eval_imgs) = (categories) * (area_ranges) * (images)
# For COCO 2017: len(eval_images) = 80 * 4 * 5000 = 1600000
self.eval_imgs = defaultdict(list)
self.eval_imgs: Any = defaultdict(list)
# Dictionary of accumulated results
self.results = {}
self.results: dict[str, Any] = {}
# Dictionary of targets for evaluation
self._targets = defaultdict(list)
self._predictions = defaultdict(list)
self._targets: defaultdict[tuple[int, int], list[Any]] = defaultdict(list)
self._predictions: defaultdict[tuple[int, int], list[Any]] = defaultdict(list)
# Parameters for evaluation
self.params = COCOEvaluatorParameters()
# List of results summarization
self.stats = []
self.stats: list[Any] = []
# Dictionary of IOUs between all targets and predictions
self.ious = {}
self.ious: dict[tuple[int, int], Any] = {}
# Set image and category ids
self.params.img_ids = sorted(self.coco_targets.get_image_ids())
self.params.cat_ids = sorted(self.coco_targets.get_category_ids())
def _prepare_targets_and_predictions(self):
def _prepare_targets_and_predictions(self) -> None:
"""
Prepare targets and predictions for evaluation.
"""
@ -602,11 +636,11 @@ class COCOEvaluator:
category.
Args:
img_id (int): The image id.
cat_id (int): The category id.
img_id: The image id.
cat_id: The category id.
Returns:
np.ndarray: The IoU between the targets and predictions.
The IoU between the targets and predictions.
"""
gt = self._targets[img_id, cat_id]
@ -614,7 +648,8 @@ class COCOEvaluator:
# If there is nothing to evaluate
if len(gt) == 0 and len(dt) == 0:
return np.array([])
empty_result: np.ndarray = np.array([], dtype=np.float64)
return empty_result
# Sort predictions by highest score first
inds = np.argsort([-d["score"] for d in dt], kind="stable")
@ -629,28 +664,32 @@ class COCOEvaluator:
dt_boxes = [d["bbox"] for d in dt]
# Get the iscrowd flag for each gt
is_crowd = [int(o["iscrowd"]) for o in gt]
is_crowd = [bool(o["iscrowd"]) for o in gt]
# Compute iou between each prediction a and gt region
iou = box_iou_batch_with_jaccard(gt_boxes, dt_boxes, is_crowd)
return iou
def _evaluate_image(
self, img_id: int, cat_id: int, area_range: tuple[int, int], max_det: int
self,
img_id: int,
cat_id: int,
area_range: list[float] | tuple[float, float],
max_det: int,
) -> dict[str, Any] | None:
"""
Perform evaluation for single category and image.
Args:
img_id (int): The image id.
cat_id (int): The category id.
area_range (Tuple[int, int]): The area range.
max_det (int): The maximum number of detections.
img_id: The image id.
cat_id: The category id.
area_range: The area range.
max_det: The maximum number of detections.
Returns:
Dict[str, Any]: The evaluation results.
The evaluation results.
"""
# Get targets (gt) and predictions (dt) for the given image and category
gt = self._targets[img_id, cat_id]
dt = self._predictions[img_id, cat_id]
gt: list[dict[str, Any]] = self._targets[img_id, cat_id]
dt: list[dict[str, Any]] = self._predictions[img_id, cat_id]
# If there is nothing to evaluate
if len(gt) == 0 and len(dt) == 0:
@ -705,7 +744,7 @@ class COCOEvaluator:
for g_idx, g in enumerate(gt):
# If current gt is already matched, and not a crowd, continue
# if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd[g_idx]:
iscrowd = int(g.get("iscrowd"))
iscrowd = int(g.get("iscrowd", 0))
if gt_matches[tresh_idx, g_idx] > 0 and not iscrowd:
continue
# Stop searching the ground truths
@ -755,10 +794,7 @@ class COCOEvaluator:
"dtIgnore": dt_ignore,
}
def __str__(self):
self.summarize()
def _accumulate(self):
def _accumulate(self) -> None:
"""
Accumulate per image evaluation results and store the result in self.results
"""
@ -796,7 +832,9 @@ class COCOEvaluator:
# Create sets for indexing
set_categories = set(self.params.cat_ids)
set_area_ranges = set(map(tuple, self.params.area_range))
set_area_ranges: set[tuple[float, ...]] = {
tuple(a) for a in self.params.area_range
}
set_max_detections = set(self.params.max_dets)
set_image_ids = set(self.params.img_ids)
@ -885,9 +923,9 @@ class COCOEvaluator:
# Precision: TP / (FP + TP)
pr = (tp / (fp + tp + EPS)).tolist()
# List to compute the precision at each recall threshold
precision_at_recall = [0] * num_recall_thresholds
precision_at_recall = [0.0] * num_recall_thresholds
# List to compute the score at each recall threshold
score_at_recall = [0] * num_recall_thresholds
score_at_recall = [0.0] * num_recall_thresholds
# Set recall to either the final recall value or 0 (when there
# is no TP)
@ -934,7 +972,9 @@ class COCOEvaluator:
}
# Helper function to compute average precision while handling -1 sentinel values
def compute_average_precision(precision_slice):
def compute_average_precision(
precision_slice: np.ndarray,
) -> tuple[np.ndarray, np.ndarray]:
"""Compute average precision while handling -1 sentinel values."""
masked = np.ma.masked_equal(precision_slice, -1)
if masked.count() == 0:
@ -1010,14 +1050,17 @@ class COCOEvaluator:
"ap_per_class_large": ap_per_class_large,
}
def _pycocotools_summarize(self):
def _pycocotools_summarize(self) -> None:
"""
Compute and display summary metrics for evaluation results.
"""
def _summarize(
use_ap: bool = True, iou_thr=None, area_range=ObjectSize.ALL, max_dets=100
):
use_ap: bool = True,
iou_thr: float | None = None,
area_range: ObjectSize = ObjectSize.ALL,
max_dets: int = 100,
) -> float:
iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.10f}"
titleStr = "Average Precision" if use_ap else "Average Recall"
typeStr = "(AP)" if use_ap else "(AR)"
@ -1047,14 +1090,14 @@ class COCOEvaluator:
s = s[t]
s = s[:, :, area_range_idx, max_detections_idx]
if len(s[s > -1]) == 0:
mean_s = -1
mean_s = -1.0
else:
mean_s = np.mean(s[s > -1])
mean_s = float(np.mean(s[s > -1]))
print(iStr.format(titleStr, typeStr, iou_str, area_range, max_dets, mean_s))
return mean_s
def _summarize_predictions():
stats = np.zeros((12,))
def _summarize_predictions() -> np.ndarray:
stats: np.ndarray = np.zeros((12,))
stats[0] = _summarize(use_ap=True)
stats[1] = _summarize(
use_ap=True, iou_thr=0.5, max_dets=self.params.max_dets[2]
@ -1098,9 +1141,9 @@ class COCOEvaluator:
return stats
if len(self.results) != 0:
self.stats = _summarize_predictions()
self.stats = _summarize_predictions().tolist()
def evaluate(self):
def evaluate(self) -> None:
"""
Start the per image evaluation on all images and keeep results in
self.eval_imgs (a list of dictionaries).
@ -1180,11 +1223,10 @@ class MeanAveragePrecision(Metric):
Initialize the Mean Average Precision metric.
Args:
metric_target (MetricTarget): The type of detection data to use.
class_agnostic (bool): Whether to treat all data as a single class.
class_mapping (Optional[Dict[int, int]]): A dictionary to map class IDs to
new IDs.
image_indices (Optional[List[int]]): The indices of the images to use.
metric_target: The type of detection data to use.
class_agnostic: Whether to treat all data as a single class.
class_mapping: A dictionary to map class IDs to new IDs.
image_indices: The indices of the images to use.
"""
self._metric_target = metric_target
self._class_agnostic = class_agnostic
@ -1210,11 +1252,11 @@ class MeanAveragePrecision(Metric):
Add new predictions and targets to the metric, but do not compute the result.
Args:
predictions (Union[Detections, List[Detections]]): The predicted detections.
targets (Union[Detections, List[Detections]]): The ground-truth detections.
predictions: The predicted detections.
targets: The ground-truth detections.
Returns:
(MeanAveragePrecision): The updated metric instance.
The updated metric instance.
"""
if not isinstance(predictions, list):
predictions = [predictions]
@ -1232,51 +1274,67 @@ class MeanAveragePrecision(Metric):
targets = deepcopy(targets)
for prediction in predictions:
prediction.class_id[:] = -1
if prediction.class_id is not None:
prediction.class_id[:] = -1
for target in targets:
target.class_id[:] = -1
if target.class_id is not None:
target.class_id[:] = -1
self._predictions_list.extend(predictions)
self._targets_list.extend(targets)
return self
def _prepare_targets(self, targets):
def _prepare_targets(
self, targets: list[Detections]
) -> dict[str, list[dict[str, Any]]]:
"""Transform targets into a dictionary that can be used by the COCO evaluator"""
images = [{"id": img_id} for img_id in range(len(targets))]
if self._image_indices is not None:
images = [
{"id": self._image_indices[img_id.get("id")]} for img_id in images
]
images = [{"id": self._image_indices[img["id"]]} for img in images]
# Annotations list
annotations = []
annotations: list[dict[str, Any]] = []
for image_id, image_targets in enumerate(targets):
if self._image_indices is not None:
image_id = self._image_indices[image_id]
for target_idx, target in enumerate(image_targets):
xyxy = target[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2]
# Ensure xyxy is not None
if image_targets.xyxy is None:
continue
for target_idx, xyxy in enumerate(image_targets.xyxy):
xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]]
# Get "area" and "iscrowd" (default 0) from data
data = target[5]
if self._class_mapping is not None:
category_id = self._class_mapping[target[3].item()]
else:
category_id = target[3].item()
# Default values
category_id = 0
if image_targets.class_id is not None:
cls_id = image_targets.class_id[target_idx]
if self._class_mapping is not None:
category_id = self._class_mapping[int(cls_id)]
else:
category_id = int(cls_id)
# Use area from data if available, otherwise calculate from bbox
area = None
if image_targets.data is not None and "area" in image_targets.data:
area = float(image_targets.data["area"][target_idx])
# Use area from data if available (e.g., COCO datasets)
# Otherwise use Detections.area property
area = data.get("area") if data else None
if area is None:
area = image_targets.area[target_idx]
area = xywh[2] * xywh[3]
iscrowd = 0
if image_targets.data is not None and "iscrowd" in image_targets.data:
iscrowd = int(image_targets.data["iscrowd"][target_idx])
dict_annotation = {
"area": area,
"iscrowd": data.get("iscrowd", 0),
"iscrowd": iscrowd,
"image_id": image_id,
"bbox": xywh,
"category_id": category_id,
"id": len(annotations) + 1, # Start IDs from 1 (0 means no match)
"ignore": 0,
}
annotations.append(dict_annotation)
# Category list
@ -1289,25 +1347,53 @@ class MeanAveragePrecision(Metric):
"categories": categories,
}
def _prepare_predictions(self, predictions):
def _prepare_predictions(
self, predictions: list[Detections]
) -> list[dict[str, Any]]:
"""Transform predictions into a list of predictions that can be used by the COCO
evaluator."""
coco_predictions = []
coco_predictions: list[dict[str, Any]] = []
for image_id, image_predictions in enumerate(predictions):
if self._image_indices is not None:
image_id = self._image_indices[image_id]
for prediction in image_predictions:
xyxy = prediction[0] # or xyxy = prediction[0]; xyxy[2:4] -= xyxy[0:2]
if image_predictions.xyxy is None:
continue
for pred_idx, xyxy in enumerate(image_predictions.xyxy):
xywh = [xyxy[0], xyxy[1], xyxy[2] - xyxy[0], xyxy[3] - xyxy[1]]
if self._class_mapping is not None:
category_id = self._class_mapping[prediction[3].item()]
else:
category_id = prediction[3].item()
category_id = 0
score = 0.0
if image_predictions.class_id is not None:
cls_id = image_predictions.class_id[pred_idx]
if self._class_mapping is not None:
category_id = self._class_mapping[int(cls_id)]
else:
category_id = int(cls_id)
if image_predictions.confidence is not None:
score = float(image_predictions.confidence[pred_idx])
# Use area from data if available, otherwise calculate from bbox
area = None
if (
image_predictions.data is not None
and "area" in image_predictions.data
):
area = float(image_predictions.data["area"][pred_idx])
if area is None:
area = xywh[2] * xywh[3]
dict_prediction = {
"image_id": image_id,
"bbox": xywh,
"score": prediction[2].item(),
"score": score,
"category_id": category_id,
"area": area,
"id": len(coco_predictions) + 1,
}
coco_predictions.append(dict_prediction)
return coco_predictions
@ -1319,7 +1405,7 @@ class MeanAveragePrecision(Metric):
Source: https://github.com/rafaelpadilla/review_object_detection_metrics
Returns:
(MeanAveragePrecisionResult): The Mean Average Precision result.
The Mean Average Precision result.
"""
total_images_predictions = len(self._predictions_list)
total_images_targets = len(self._targets_list)
@ -1349,7 +1435,7 @@ class MeanAveragePrecision(Metric):
mAP_scores=cocoEval.results["mAP_scores_small"],
ap_per_class=cocoEval.results["ap_per_class_small"],
iou_thresholds=cocoEval.params.iou_thrs,
matched_classes=cocoEval.params.cat_ids,
matched_classes=np.array(cocoEval.params.cat_ids),
)
# Create MeanAveragePrecisionResult object for medium objects
mAP_medium = MeanAveragePrecisionResult(
@ -1358,7 +1444,7 @@ class MeanAveragePrecision(Metric):
mAP_scores=cocoEval.results["mAP_scores_medium"],
ap_per_class=cocoEval.results["ap_per_class_medium"],
iou_thresholds=cocoEval.params.iou_thrs,
matched_classes=cocoEval.params.cat_ids,
matched_classes=np.array(cocoEval.params.cat_ids),
)
# Create MeanAveragePrecisionResult object for large objects
mAP_large = MeanAveragePrecisionResult(
@ -1367,7 +1453,7 @@ class MeanAveragePrecision(Metric):
mAP_scores=cocoEval.results["mAP_scores_large"],
ap_per_class=cocoEval.results["ap_per_class_large"],
iou_thresholds=cocoEval.params.iou_thrs,
matched_classes=cocoEval.params.cat_ids,
matched_classes=np.array(cocoEval.params.cat_ids),
)
# Create the final MeanAveragePrecisionResult object
@ -1377,7 +1463,7 @@ class MeanAveragePrecision(Metric):
mAP_scores=cocoEval.results["mAP_scores_all_sizes"],
ap_per_class=cocoEval.results["ap_per_class_all_sizes"],
iou_thresholds=cocoEval.params.iou_thrs,
matched_classes=cocoEval.params.cat_ids,
matched_classes=np.array(cocoEval.params.cat_ids),
small_objects=mAP_small,
medium_objects=mAP_medium,
large_objects=mAP_large,

518
supervision/metrics/mean_average_recall.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import numpy as np
from matplotlib import pyplot as plt
@ -26,6 +26,218 @@ if TYPE_CHECKING:
import pandas as pd
@dataclass
class MeanAverageRecallResult:
"""
The results of the Mean Average Recall metric calculation.
Defaults to `0` if no detections or targets were provided.
Attributes:
metric_target: the type of data used for the metric -
boxes, masks or oriented bounding boxes.
mAR_at_1: the Mean Average Recall, when considering only the top
highest confidence detection for each class.
mAR_at_10: the Mean Average Recall, when considering top 10
highest confidence detections for each class.
mAR_at_100: the Mean Average Recall, when considering top 100
highest confidence detections for each class.
recall_per_class: the recall scores per class and IoU threshold.
Shape: `(num_target_classes, num_iou_thresholds)`
max_detections: the array with maximum number of detections
considered.
iou_thresholds: the IoU thresholds used in the calculations.
matched_classes: the class IDs of all matched classes.
Corresponds to the rows of `recall_per_class`.
small_objects: the Mean Average Recall
metric results for small objects (area < 32²).
medium_objects: the Mean Average Recall
metric results for medium objects (32² ≤ area < 96²).
large_objects: the Mean Average Recall
metric results for large objects (area ≥ 96²).
"""
metric_target: MetricTarget
@property
def mAR_at_1(self) -> float:
return float(self.recall_scores[0])
@property
def mAR_at_10(self) -> float:
return float(self.recall_scores[1])
@property
def mAR_at_100(self) -> float:
return float(self.recall_scores[2])
recall_scores: np.ndarray
recall_per_class: np.ndarray
max_detections: np.ndarray
iou_thresholds: np.ndarray
matched_classes: np.ndarray
small_objects: MeanAverageRecallResult | None
medium_objects: MeanAverageRecallResult | None
large_objects: MeanAverageRecallResult | None
def __str__(self) -> str:
"""
Format as a pretty string.
Example:
```python
print(mar_results)
# MeanAverageRecallResult:
# Metric target: MetricTarget.BOXES
# mAR @ 1: 0.1362
# mAR @ 10: 0.4239
# mAR @ 100: 0.5241
# max detections: [1 10 100]
# IoU thresh: [0.5 0.55 0.6 ...]
# mAR per class:
# 0: [0.78571 0.78571 0.78571 ...]
# ...
# Small objects: ...
# Medium objects: ...
# Large objects: ...
```
"""
out_str = (
f"{self.__class__.__name__}:\n"
f"Metric target: {self.metric_target}\n"
f"mAR @ 1: {self.mAR_at_1:.4f}\n"
f"mAR @ 10: {self.mAR_at_10:.4f}\n"
f"mAR @ 100: {self.mAR_at_100:.4f}\n"
f"max detections: {self.max_detections}\n"
f"IoU thresh: {self.iou_thresholds}\n"
f"mAR per class:\n"
)
if self.recall_per_class.size == 0:
out_str += " No results\n"
for class_id, recall_of_class in zip(
self.matched_classes, self.recall_per_class
):
out_str += f" {class_id}: {recall_of_class}\n"
indent = " "
if self.small_objects is not None:
indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
out_str += f"\nSmall objects:\n{indented}"
if self.medium_objects is not None:
indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
out_str += f"\nMedium objects:\n{indented}"
if self.large_objects is not None:
indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
out_str += f"\nLarge objects:\n{indented}"
return out_str
def to_pandas(self) -> pd.DataFrame:
"""
Convert the result to a pandas DataFrame.
Returns:
The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
pandas_data = {
"mAR @ 1": self.mAR_at_1,
"mAR @ 10": self.mAR_at_10,
"mAR @ 100": self.mAR_at_100,
}
if self.small_objects is not None:
small_objects_df = self.small_objects.to_pandas()
for key, value in small_objects_df.items():
pandas_data[f"small_objects_{key}"] = value
if self.medium_objects is not None:
medium_objects_df = self.medium_objects.to_pandas()
for key, value in medium_objects_df.items():
pandas_data[f"medium_objects_{key}"] = value
if self.large_objects is not None:
large_objects_df = self.large_objects.to_pandas()
for key, value in large_objects_df.items():
pandas_data[f"large_objects_{key}"] = value
return pd.DataFrame(pandas_data, index=[0])
def plot(self) -> None:
"""
Plot the Mean Average Recall results.
![example_plot](\
https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png\
){ align=center width="800" }
"""
labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"]
values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100]
colors = [LEGACY_COLOR_PALETTE[0]] * 3
if self.small_objects is not None:
small_objects = self.small_objects
labels += ["Small: mAR @ 1", "Small: mAR @ 10", "Small: mAR @ 100"]
values += [
small_objects.mAR_at_1,
small_objects.mAR_at_10,
small_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[3]] * 3
if self.medium_objects is not None:
medium_objects = self.medium_objects
labels += ["Medium: mAR @ 1", "Medium: mAR @ 10", "Medium: mAR @ 100"]
values += [
medium_objects.mAR_at_1,
medium_objects.mAR_at_10,
medium_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[2]] * 3
if self.large_objects is not None:
large_objects = self.large_objects
labels += ["Large: mAR @ 1", "Large: mAR @ 10", "Large: mAR @ 100"]
values += [
large_objects.mAR_at_1,
large_objects.mAR_at_10,
large_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[4]] * 3
plt.rcParams["font.family"] = "monospace"
_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"Mean Average Recall, by Object Size\n(target: {self.metric_target.value})"
)
ax.set_title(title, fontweight="bold")
x_positions = range(len(labels))
bars = ax.bar(x_positions, values, color=colors, align="center")
ax.set_xticks(x_positions)
ax.set_xticklabels(labels, rotation=45, ha="right")
for bar in bars:
y_value = bar.get_height()
ax.text(
bar.get_x() + bar.get_width() / 2,
y_value + 0.02,
f"{y_value:.2f}",
ha="center",
va="bottom",
)
plt.rcParams["font.family"] = "sans-serif"
plt.tight_layout()
plt.show()
class MeanAverageRecall(Metric):
"""
Mean Average Recall (mAR) measures how well the model detects
@ -69,7 +281,7 @@ class MeanAverageRecall(Metric):
Initialize the Mean Average Recall metric.
Args:
metric_target (MetricTarget): The type of detection data to use.
metric_target: The type of detection data to use.
"""
self._metric_target = metric_target
@ -94,11 +306,11 @@ class MeanAverageRecall(Metric):
Add new predictions and targets to the metric, but do not compute the result.
Args:
predictions (Union[Detections, List[Detections]]): The predicted detections.
targets (Union[Detections, List[Detections]]): The target detections.
predictions: The predicted detections.
targets: The target detections.
Returns:
(Recall): The updated metric instance.
The updated metric instance.
"""
if not isinstance(predictions, list):
predictions = [predictions]
@ -122,7 +334,7 @@ class MeanAverageRecall(Metric):
and ground-truth, at different IoU thresholds and maximum detection counts.
Returns:
(MeanAverageRecallResult): The Mean Average Recall metric result.
The Mean Average Recall metric result.
"""
result = self._compute(self._predictions_list, self._targets_list)
@ -149,7 +361,7 @@ class MeanAverageRecall(Metric):
self, predictions_list: list[Detections], targets_list: list[Detections]
) -> MeanAverageRecallResult:
iou_thresholds = np.linspace(0.5, 0.95, 10)
stats = []
stats: list[Any] = []
for predictions, targets in zip(predictions_list, targets_list):
prediction_contents = self._detections_content(predictions)
@ -181,7 +393,14 @@ class MeanAverageRecall(Metric):
)
matches = self._match_detection_batch(
predictions.class_id, targets.class_id, iou, iou_thresholds
predictions.class_id
if predictions.class_id is not None
else np.array([]),
targets.class_id
if targets.class_id is not None
else np.array([]),
iou,
iou_thresholds,
)
stats.append(
(
@ -286,8 +505,8 @@ class MeanAverageRecall(Metric):
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
correct[matches[:, 1].astype(int), i] = True
return correct
result: np.ndarray = correct
return result
@staticmethod
def _compute_confusion_matrix(
@ -304,20 +523,20 @@ class MeanAverageRecall(Metric):
in descending order.
Args:
sorted_matches: np.ndarray, bool, shape (P, Th), that is True
sorted_matches: shape (P, Th), that is True
if the prediction is a true positive at the given IoU threshold.
sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
sorted_prediction_class_ids: shape (P,), containing
the class id for each prediction.
unique_classes: np.ndarray, int, shape (C,), containing the unique
unique_classes: shape (C,), containing the unique
class ids.
class_counts: np.ndarray, int, shape (C,), containing the number
class_counts: shape (C,), containing the number
of true instances for each class.
max_detections: Optional[int], the maximum number of detections to
max_detections: The maximum number of detections to
consider for each class. Extra detections are considered false
positives. By default, all detections are considered.
Returns:
np.ndarray, shape (C, Th, 3), containing the true positives, false
shape (C, Th, 3), containing the true positives, false
positives, and false negatives for each class and IoU threshold.
"""
num_thresholds = sorted_matches.shape[1]
@ -343,12 +562,13 @@ class MeanAverageRecall(Metric):
false_positives = (1 - limited_matches).sum(0)
false_negatives = num_true - true_positives
false_negatives = num_true - true_positives
confusion_matrix[class_idx] = np.stack(
[true_positives, false_positives, false_negatives], axis=1
)
return confusion_matrix
result_confusion_matrix: np.ndarray = confusion_matrix
return result_confusion_matrix
@staticmethod
def _compute_recall(confusion_matrix: np.ndarray) -> np.ndarray:
@ -356,11 +576,11 @@ class MeanAverageRecall(Metric):
Broadcastable function, computing the recall from the confusion matrix.
Arguments:
confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
confusion_matrix: shape (N, ..., 3), where the last dimension
contains the true positives, false positives, and false negatives.
Returns:
np.ndarray, shape (N, ...), containing the recall for each element.
shape (N, ...), containing the recall for each element.
"""
if not confusion_matrix.shape[-1] == 3:
raise ValueError(
@ -373,7 +593,8 @@ class MeanAverageRecall(Metric):
denominator = true_positives + false_negatives
recall = np.where(denominator == 0, 0, true_positives / denominator)
return recall
result_recall: np.ndarray = recall
return result_recall
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@ -388,17 +609,24 @@ class MeanAverageRecall(Metric):
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
obb = detections.data.get(ORIENTED_BOX_COORDINATES)
if obb is not None and len(obb) > 0:
return np.array(obb, dtype=np.float32)
result_obb: np.ndarray = np.array(obb, dtype=np.float32)
return result_obb
return self._make_empty_content()
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _make_empty_content(self) -> np.ndarray:
if self._metric_target == MetricTarget.BOXES:
return np.empty((0, 4), dtype=np.float32)
empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32)
return empty_boxes
if self._metric_target == MetricTarget.MASKS:
return np.empty((0, 0, 0), dtype=bool)
empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool)
return empty_masks
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
return np.empty((0, 4, 2), dtype=np.float32)
empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32)
return empty_obb
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _filter_detections_by_size(
@ -433,6 +661,9 @@ class MeanAverageRecall(Metric):
targets_list: list[Detections],
size_category: ObjectSizeCategory,
) -> tuple[list[Detections], list[Detections]]:
"""
Filter predictions and targets by object size category.
"""
new_predictions_list = []
new_targets_list = []
for predictions, targets in zip(predictions_list, targets_list):
@ -443,240 +674,3 @@ class MeanAverageRecall(Metric):
self._filter_detections_by_size(targets, size_category)
)
return new_predictions_list, new_targets_list
@dataclass
class MeanAverageRecallResult:
# """
# The results of the recall metric calculation.
# Defaults to `0` if no detections or targets were provided.
# Attributes:
# metric_target (MetricTarget): the type of data used for the metric -
# boxes, masks or oriented bounding boxes.
# averaging_method (AveragingMethod): the averaging method used to compute the
# recall. Determines how the recall is aggregated across classes.
# recall_at_50 (float): the recall at IoU threshold of `0.5`.
# recall_at_75 (float): the recall at IoU threshold of `0.75`.
# recall_scores (np.ndarray): the recall scores at each IoU threshold.
# Shape: `(num_iou_thresholds,)`
# recall_per_class (np.ndarray): the recall scores per class and IoU threshold.
# Shape: `(num_target_classes, num_iou_thresholds)`
# iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
# matched_classes (np.ndarray): the class IDs of all matched classes.
# Corresponds to the rows of `recall_per_class`.
# small_objects (Optional[RecallResult]): the Recall metric results
# for small objects.
# medium_objects (Optional[RecallResult]): the Recall metric results
# for medium objects.
# large_objects (Optional[RecallResult]): the Recall metric results
# for large objects.
# """
"""
The results of the Mean Average Recall metric calculation.
Defaults to `0` if no detections or targets were provided.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
boxes, masks or oriented bounding boxes.
mAR_at_1 (float): the Mean Average Recall, when considering only the top
highest confidence detection for each class.
mAR_at_10 (float): the Mean Average Recall, when considering top 10
highest confidence detections for each class.
mAR_at_100 (float): the Mean Average Recall, when considering top 100
highest confidence detections for each class.
recall_per_class (np.ndarray): the recall scores per class and IoU threshold.
Shape: `(num_target_classes, num_iou_thresholds)`
max_detections (np.ndarray): the array with maximum number of detections
considered.
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
Corresponds to the rows of `recall_per_class`.
small_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall
metric results for small objects (area < 32²).
medium_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall
metric results for medium objects (32² ≤ area < 96²).
large_objects (Optional[MeanAverageRecallResult]): the Mean Average Recall
metric results for large objects (area ≥ 96²).
"""
metric_target: MetricTarget
@property
def mAR_at_1(self) -> float:
return self.recall_scores[0]
@property
def mAR_at_10(self) -> float:
return self.recall_scores[1]
@property
def mAR_at_100(self) -> float:
return self.recall_scores[2]
recall_scores: np.ndarray
recall_per_class: np.ndarray
max_detections: np.ndarray
iou_thresholds: np.ndarray
matched_classes: np.ndarray
small_objects: MeanAverageRecallResult | None
medium_objects: MeanAverageRecallResult | None
large_objects: MeanAverageRecallResult | None
def __str__(self) -> str:
"""
Format as a pretty string.
Example:
```python
# MeanAverageRecallResult:
# Metric target: MetricTarget.BOXES
# mAR @ 1: 0.1362
# mAR @ 10: 0.4239
# mAR @ 100: 0.5241
# max detections: [1 10 100]
# IoU thresh: [0.5 0.55 0.6 ...]
# mAR per class:
# 0: [0.78571 0.78571 0.78571 ...]
# ...
# Small objects: ...
# Medium objects: ...
# Large objects: ...
```
"""
out_str = (
f"{self.__class__.__name__}:\n"
f"Metric target: {self.metric_target}\n"
f"mAR @ 1: {self.mAR_at_1:.4f}\n"
f"mAR @ 10: {self.mAR_at_10:.4f}\n"
f"mAR @ 100: {self.mAR_at_100:.4f}\n"
f"max detections: {self.max_detections}\n"
f"IoU thresh: {self.iou_thresholds}\n"
f"mAR per class:\n"
)
if self.recall_per_class.size == 0:
out_str += " No results\n"
for class_id, recall_of_class in zip(
self.matched_classes, self.recall_per_class
):
out_str += f" {class_id}: {recall_of_class}\n"
indent = " "
if self.small_objects is not None:
indented = indent + str(self.small_objects).replace("\n", f"\n{indent}")
out_str += f"\nSmall objects:\n{indented}"
if self.medium_objects is not None:
indented = indent + str(self.medium_objects).replace("\n", f"\n{indent}")
out_str += f"\nMedium objects:\n{indented}"
if self.large_objects is not None:
indented = indent + str(self.large_objects).replace("\n", f"\n{indent}")
out_str += f"\nLarge objects:\n{indented}"
return out_str
def to_pandas(self) -> pd.DataFrame:
"""
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
pandas_data = {
"mAR @ 1": self.mAR_at_1,
"mAR @ 10": self.mAR_at_10,
"mAR @ 100": self.mAR_at_100,
}
if self.small_objects is not None:
small_objects_df = self.small_objects.to_pandas()
for key, value in small_objects_df.items():
pandas_data[f"small_objects_{key}"] = value
if self.medium_objects is not None:
medium_objects_df = self.medium_objects.to_pandas()
for key, value in medium_objects_df.items():
pandas_data[f"medium_objects_{key}"] = value
if self.large_objects is not None:
large_objects_df = self.large_objects.to_pandas()
for key, value in large_objects_df.items():
pandas_data[f"large_objects_{key}"] = value
return pd.DataFrame(pandas_data, index=[0])
def plot(self):
"""
Plot the Mean Average Recall results.
![example_plot](
https://media.roboflow.com/supervision-docs/metrics/mAR_plot_example.png
){ align=center width="800" }
"""
labels = ["mAR @ 1", "mAR @ 10", "mAR @ 100"]
values = [self.mAR_at_1, self.mAR_at_10, self.mAR_at_100]
colors = [LEGACY_COLOR_PALETTE[0]] * 3
if self.small_objects is not None:
small_objects = self.small_objects
labels += ["Small: mAR @ 1", "Small: mAR @ 10", "Small: mAR @ 100"]
values += [
small_objects.mAR_at_1,
small_objects.mAR_at_10,
small_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[3]] * 3
if self.medium_objects is not None:
medium_objects = self.medium_objects
labels += ["Medium: mAR @ 1", "Medium: mAR @ 10", "Medium: mAR @ 100"]
values += [
medium_objects.mAR_at_1,
medium_objects.mAR_at_10,
medium_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[2]] * 3
if self.large_objects is not None:
large_objects = self.large_objects
labels += ["Large: mAR @ 1", "Large: mAR @ 10", "Large: mAR @ 100"]
values += [
large_objects.mAR_at_1,
large_objects.mAR_at_10,
large_objects.mAR_at_100,
]
colors += [LEGACY_COLOR_PALETTE[4]] * 3
plt.rcParams["font.family"] = "monospace"
_, ax = plt.subplots(figsize=(10, 6))
ax.set_ylim(0, 1)
ax.set_ylabel("Value", fontweight="bold")
title = (
f"Mean Average Recall, by Object Size\n(target: {self.metric_target.value})"
)
ax.set_title(title, fontweight="bold")
x_positions = range(len(labels))
bars = ax.bar(x_positions, values, color=colors, align="center")
ax.set_xticks(x_positions)
ax.set_xticklabels(labels, rotation=45, ha="right")
for bar in bars:
y_value = bar.get_height()
ax.text(
bar.get_x() + bar.get_width() / 2,
y_value + 0.02,
f"{y_value:.2f}",
ha="center",
va="bottom",
)
plt.rcParams["font.family"] = "sans-serif"
plt.tight_layout()
plt.show()

97
supervision/metrics/precision.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import numpy as np
from matplotlib import pyplot as plt
@ -71,8 +71,8 @@ class Precision(Metric):
Initialize the Precision metric.
Args:
metric_target (MetricTarget): The type of detection data to use.
averaging_method (AveragingMethod): The averaging method used to compute the
metric_target: The type of detection data to use.
averaging_method: The averaging method used to compute the
precision. Determines how the precision is aggregated across classes.
"""
self._metric_target = metric_target
@ -97,11 +97,11 @@ class Precision(Metric):
Add new predictions and targets to the metric, but do not compute the result.
Args:
predictions (Union[Detections, List[Detections]]): The predicted detections.
targets (Union[Detections, List[Detections]]): The target detections.
predictions: The predicted detections.
targets: The target detections.
Returns:
(Precision): The updated metric instance.
The updated metric instance.
"""
if not isinstance(predictions, list):
predictions = [predictions]
@ -125,7 +125,7 @@ class Precision(Metric):
data, at different IoU thresholds.
Returns:
(PrecisionResult): The precision metric result.
The precision metric result.
"""
result = self._compute(self._predictions_list, self._targets_list)
@ -152,7 +152,7 @@ class Precision(Metric):
self, predictions_list: list[Detections], targets_list: list[Detections]
) -> PrecisionResult:
iou_thresholds = np.linspace(0.5, 0.95, 10)
stats = []
stats: list[Any] = []
for predictions, targets in zip(predictions_list, targets_list):
prediction_contents = self._detections_content(predictions)
@ -184,7 +184,14 @@ class Precision(Metric):
)
matches = self._match_detection_batch(
predictions.class_id, targets.class_id, iou, iou_thresholds
predictions.class_id
if predictions.class_id is not None
else np.array([]),
targets.class_id
if targets.class_id is not None
else np.array([]),
iou,
iou_thresholds,
)
stats.append(
(
@ -287,8 +294,8 @@ class Precision(Metric):
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
correct[matches[:, 1].astype(int), i] = True
return correct
result: np.ndarray = correct
return result
@staticmethod
def _compute_confusion_matrix(
@ -303,18 +310,18 @@ class Precision(Metric):
Assumes the matches and prediction_class_ids are sorted by confidence
in descending order.
Arguments:
sorted_matches: np.ndarray, bool, shape (P, Th), that is True
Args:
sorted_matches: shape (P, Th), that is True
if the prediction is a true positive at the given IoU threshold.
sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
sorted_prediction_class_ids: shape (P,), containing
the class id for each prediction.
unique_classes: np.ndarray, int, shape (C,), containing the unique
unique_classes: shape (C,), containing the unique
class ids.
class_counts: np.ndarray, int, shape (C,), containing the number
class_counts: shape (C,), containing the number
of true instances for each class.
Returns:
np.ndarray, shape (C, Th, 3), containing the true positives, false
shape (C, Th, 3), containing the true positives, false
positives, and false negatives for each class and IoU threshold.
"""
@ -342,8 +349,8 @@ class Precision(Metric):
confusion_matrix[class_idx] = np.stack(
[true_positives, false_positives, false_negatives], axis=1
)
return confusion_matrix
result_matrix: np.ndarray = confusion_matrix
return result_matrix
@staticmethod
def _compute_precision(confusion_matrix: np.ndarray) -> np.ndarray:
@ -351,11 +358,11 @@ class Precision(Metric):
Broadcastable function, computing the precision from the confusion matrix.
Arguments:
confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
confusion_matrix: shape (N, ..., 3), where the last dimension
contains the true positives, false positives, and false negatives.
Returns:
np.ndarray, shape (N, ...), containing the precision for each element.
shape (N, ...), containing the precision for each element.
"""
if not confusion_matrix.shape[-1] == 3:
raise ValueError(
@ -368,7 +375,8 @@ class Precision(Metric):
denominator = true_positives + false_positives
precision = np.where(denominator == 0, 0, true_positives / denominator)
return precision
result_precision: np.ndarray = precision
return result_precision
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@ -383,17 +391,24 @@ class Precision(Metric):
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
obb = detections.data.get(ORIENTED_BOX_COORDINATES)
if obb is not None and len(obb) > 0:
return np.array(obb, dtype=np.float32)
result_obb: np.ndarray = np.array(obb, dtype=np.float32)
return result_obb
return self._make_empty_content()
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _make_empty_content(self) -> np.ndarray:
if self._metric_target == MetricTarget.BOXES:
return np.empty((0, 4), dtype=np.float32)
empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32)
return empty_boxes
if self._metric_target == MetricTarget.MASKS:
return np.empty((0, 0, 0), dtype=bool)
empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool)
return empty_masks
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
return np.empty((0, 4, 2), dtype=np.float32)
empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32)
return empty_obb
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _filter_detections_by_size(
@ -451,24 +466,24 @@ class PrecisionResult:
Defaults to `0` if no detections or targets were provided.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
metric_target: the type of data used for the metric -
boxes, masks or oriented bounding boxes.
averaging_method (AveragingMethod): the averaging method used to compute the
averaging_method: the averaging method used to compute the
precision. Determines how the precision is aggregated across classes.
precision_at_50 (float): the precision at IoU threshold of `0.5`.
precision_at_75 (float): the precision at IoU threshold of `0.75`.
precision_scores (np.ndarray): the precision scores at each IoU threshold.
precision_at_50: the precision at IoU threshold of `0.5`.
precision_at_75: the precision at IoU threshold of `0.75`.
precision_scores: the precision scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
precision_per_class (np.ndarray): the precision scores per class and
precision_per_class: the precision scores per class and
IoU threshold. Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
iou_thresholds: the IoU thresholds used in the calculations.
matched_classes: the class IDs of all matched classes.
Corresponds to the rows of `precision_per_class`.
small_objects (Optional[PrecisionResult]): the Precision metric results
small_objects: the Precision metric results
for small objects (area < 32²).
medium_objects (Optional[PrecisionResult]): the Precision metric results
medium_objects: the Precision metric results
for medium objects (32² ≤ area < 96²).
large_objects (Optional[PrecisionResult]): the Precision metric results
large_objects: the Precision metric results
for large objects (area ≥ 96²).
"""
@ -477,11 +492,11 @@ class PrecisionResult:
@property
def precision_at_50(self) -> float:
return self.precision_scores[0]
return float(self.precision_scores[0])
@property
def precision_at_75(self) -> float:
return self.precision_scores[5]
return float(self.precision_scores[5])
precision_scores: np.ndarray
precision_per_class: np.ndarray
@ -549,7 +564,7 @@ class PrecisionResult:
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
@ -574,7 +589,7 @@ class PrecisionResult:
return pd.DataFrame(pandas_data, index=[0])
def plot(self):
def plot(self) -> None:
"""
Plot the precision results.

97
supervision/metrics/recall.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from copy import deepcopy
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import numpy as np
from matplotlib import pyplot as plt
@ -71,8 +71,8 @@ class Recall(Metric):
Initialize the Recall metric.
Args:
metric_target (MetricTarget): The type of detection data to use.
averaging_method (AveragingMethod): The averaging method used to compute the
metric_target: The type of detection data to use.
averaging_method: The averaging method used to compute the
recall. Determines how the recall is aggregated across classes.
"""
self._metric_target = metric_target
@ -97,11 +97,11 @@ class Recall(Metric):
Add new predictions and targets to the metric, but do not compute the result.
Args:
predictions (Union[Detections, List[Detections]]): The predicted detections.
targets (Union[Detections, List[Detections]]): The target detections.
predictions: The predicted detections.
targets: The target detections.
Returns:
(Recall): The updated metric instance.
The updated metric instance.
"""
if not isinstance(predictions, list):
predictions = [predictions]
@ -125,7 +125,7 @@ class Recall(Metric):
data, at different IoU thresholds.
Returns:
(RecallResult): The recall metric result.
The recall metric result.
"""
result = self._compute(self._predictions_list, self._targets_list)
@ -152,7 +152,7 @@ class Recall(Metric):
self, predictions_list: list[Detections], targets_list: list[Detections]
) -> RecallResult:
iou_thresholds = np.linspace(0.5, 0.95, 10)
stats = []
stats: list[Any] = []
for predictions, targets in zip(predictions_list, targets_list):
prediction_contents = self._detections_content(predictions)
@ -184,7 +184,14 @@ class Recall(Metric):
)
matches = self._match_detection_batch(
predictions.class_id, targets.class_id, iou, iou_thresholds
predictions.class_id
if predictions.class_id is not None
else np.array([]),
targets.class_id
if targets.class_id is not None
else np.array([]),
iou,
iou_thresholds,
)
stats.append(
(
@ -285,8 +292,8 @@ class Recall(Metric):
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
correct[matches[:, 1].astype(int), i] = True
return correct
result: np.ndarray = correct
return result
@staticmethod
def _compute_confusion_matrix(
@ -301,18 +308,18 @@ class Recall(Metric):
Assumes the matches and prediction_class_ids are sorted by confidence
in descending order.
Arguments:
sorted_matches: np.ndarray, bool, shape (P, Th), that is True
Args:
sorted_matches: shape (P, Th), that is True
if the prediction is a true positive at the given IoU threshold.
sorted_prediction_class_ids: np.ndarray, int, shape (P,), containing
sorted_prediction_class_ids: shape (P,), containing
the class id for each prediction.
unique_classes: np.ndarray, int, shape (C,), containing the unique
unique_classes: shape (C,), containing the unique
class ids.
class_counts: np.ndarray, int, shape (C,), containing the number
class_counts: shape (C,), containing the number
of true instances for each class.
Returns:
np.ndarray, shape (C, Th, 3), containing the true positives, false
shape (C, Th, 3), containing the true positives, false
positives, and false negatives for each class and IoU threshold.
"""
@ -340,8 +347,8 @@ class Recall(Metric):
confusion_matrix[class_idx] = np.stack(
[true_positives, false_positives, false_negatives], axis=1
)
return confusion_matrix
result: np.ndarray = confusion_matrix
return result
@staticmethod
def _compute_recall(confusion_matrix: np.ndarray) -> np.ndarray:
@ -349,11 +356,11 @@ class Recall(Metric):
Broadcastable function, computing the recall from the confusion matrix.
Arguments:
confusion_matrix: np.ndarray, shape (N, ..., 3), where the last dimension
confusion_matrix: shape (N, ..., 3), where the last dimension
contains the true positives, false positives, and false negatives.
Returns:
np.ndarray, shape (N, ...), containing the recall for each element.
shape (N, ...), containing the recall for each element.
"""
if not confusion_matrix.shape[-1] == 3:
raise ValueError(
@ -366,7 +373,8 @@ class Recall(Metric):
denominator = true_positives + false_negatives
recall = np.where(denominator == 0, 0, true_positives / denominator)
return recall
result: np.ndarray = recall
return result
def _detections_content(self, detections: Detections) -> np.ndarray:
"""Return boxes, masks or oriented bounding boxes from detections."""
@ -381,17 +389,24 @@ class Recall(Metric):
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
obb = detections.data.get(ORIENTED_BOX_COORDINATES)
if obb is not None and len(obb) > 0:
return np.array(obb, dtype=np.float32)
result: np.ndarray = np.array(obb, dtype=np.float32)
return result
return self._make_empty_content()
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _make_empty_content(self) -> np.ndarray:
if self._metric_target == MetricTarget.BOXES:
return np.empty((0, 4), dtype=np.float32)
empty_boxes: np.ndarray = np.empty((0, 4), dtype=np.float32)
return empty_boxes
if self._metric_target == MetricTarget.MASKS:
return np.empty((0, 0, 0), dtype=bool)
empty_masks: np.ndarray = np.empty((0, 0, 0), dtype=bool)
return empty_masks
if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
return np.empty((0, 4, 2), dtype=np.float32)
empty_obb: np.ndarray = np.empty((0, 4, 2), dtype=np.float32)
return empty_obb
raise ValueError(f"Invalid metric target: {self._metric_target}")
def _filter_detections_by_size(
@ -449,24 +464,24 @@ class RecallResult:
Defaults to `0` if no detections or targets were provided.
Attributes:
metric_target (MetricTarget): the type of data used for the metric -
metric_target: the type of data used for the metric -
boxes, masks or oriented bounding boxes.
averaging_method (AveragingMethod): the averaging method used to compute the
averaging_method: the averaging method used to compute the
recall. Determines how the recall is aggregated across classes.
recall_at_50 (float): the recall at IoU threshold of `0.5`.
recall_at_75 (float): the recall at IoU threshold of `0.75`.
recall_scores (np.ndarray): the recall scores at each IoU threshold.
recall_at_50: the recall at IoU threshold of `0.5`.
recall_at_75: the recall at IoU threshold of `0.75`.
recall_scores: the recall scores at each IoU threshold.
Shape: `(num_iou_thresholds,)`
recall_per_class (np.ndarray): the recall scores per class and IoU threshold.
recall_per_class: the recall scores per class and IoU threshold.
Shape: `(num_target_classes, num_iou_thresholds)`
iou_thresholds (np.ndarray): the IoU thresholds used in the calculations.
matched_classes (np.ndarray): the class IDs of all matched classes.
iou_thresholds: the IoU thresholds used in the calculations.
matched_classes: the class IDs of all matched classes.
Corresponds to the rows of `recall_per_class`.
small_objects (Optional[RecallResult]): the Recall metric results
small_objects: the Recall metric results
for small objects (area < 32²).
medium_objects (Optional[RecallResult]): the Recall metric results
medium_objects: the Recall metric results
for medium objects (32² ≤ area < 96²).
large_objects (Optional[RecallResult]): the Recall metric results
large_objects: the Recall metric results
for large objects (area ≥ 96²).
"""
@ -475,11 +490,11 @@ class RecallResult:
@property
def recall_at_50(self) -> float:
return self.recall_scores[0]
return float(self.recall_scores[0])
@property
def recall_at_75(self) -> float:
return self.recall_scores[5]
return float(self.recall_scores[5])
recall_scores: np.ndarray
recall_per_class: np.ndarray
@ -547,7 +562,7 @@ class RecallResult:
Convert the result to a pandas DataFrame.
Returns:
(pd.DataFrame): The result as a DataFrame.
The result as a DataFrame.
"""
ensure_pandas_installed()
import pandas as pd
@ -572,7 +587,7 @@ class RecallResult:
return pd.DataFrame(pandas_data, index=[0])
def plot(self):
def plot(self) -> None:
"""
Plot the recall results.

2
supervision/metrics/utils/utils.py
View File

@ -1,4 +1,4 @@
def ensure_pandas_installed():
def ensure_pandas_installed() -> None:
try:
import pandas # noqa
except ImportError: