diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py index 527febe4..74879f79 100644 --- a/supervision/metrics/core.py +++ b/supervision/metrics/core.py @@ -2,7 +2,7 @@ from __future__ import annotations from abc import ABC, abstractmethod from enum import Enum -from typing import Any, Dict, Iterator, Optional, Tuple, Union +from typing import Any, Dict, Iterator, Tuple, Union import numpy as np import numpy.typing as npt @@ -10,6 +10,7 @@ from typing_extensions import Self from supervision import config from supervision.detection.core import Detections +from supervision.metrics.utils import len0_like, pad_mask CLASS_ID_NONE = -1 """Used by metrics module as class ID, when none is present""" @@ -95,19 +96,13 @@ class InternalMetricDataStore: self._class_agnostic = class_agnostic self._data_1: Dict[int, npt.NDArray] self._data_2: Dict[int, npt.NDArray] - self._datapoint_shape: Optional[Tuple[int, ...]] + self._mask_shape: Tuple[int, int] self.reset() def reset(self) -> None: self._data_1 = {} self._data_2 = {} - if self._metric_target == MetricTarget.BOXES: - self._datapoint_shape = (4,) - elif self._metric_target == MetricTarget.MASKS: - # Determined when adding data - self._datapoint_shape = None - elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - self._datapoint_shape = (8,) + self._mask_shape = (0, 0) def update( self, @@ -116,47 +111,45 @@ class InternalMetricDataStore: ) -> None: content_1 = self._get_content(data_1) content_2 = self._get_content(data_2) + self._validate_shape(content_1) + self._validate_shape(content_2) + class_ids_1 = self._get_class_ids(data_1) class_ids_2 = self._get_class_ids(data_2) self._validate_class_ids(class_ids_1, class_ids_2) - if content_1 is not None and len(content_1) > 0: - assert len(content_1) == len(class_ids_1) - for class_id in set(class_ids_1): - content_of_class = content_1[class_ids_1 == class_id] - if class_id not in self._data_1: - self._data_1[class_id] = content_of_class - continue - self._data_1[class_id] = np.vstack( - (self._data_1[class_id], content_of_class) - ) - if content_2 is not None and len(content_2) > 0: - assert len(content_2) == len(class_ids_2) - for class_id in set(class_ids_2): - content_of_class = content_2[class_ids_2 == class_id] - if class_id not in self._data_2: - self._data_2[class_id] = content_of_class - continue - self._data_2[class_id] = np.vstack( - (self._data_2[class_id], content_of_class) - ) + assert len(content_1) == len(class_ids_1) and len(content_2) == len(class_ids_2) + + if self._metric_target == MetricTarget.MASKS: + content_1 = self._expand_mask_shape(content_1) + content_2 = self._expand_mask_shape(content_2) + + for class_id in set(class_ids_1): + content_of_class = content_1[class_ids_1 == class_id] + stored_content_of_class = self._data_1.get(class_id, len0_like(content_1)) + self._data_1[class_id] = np.vstack( + (stored_content_of_class, content_of_class) + ) + + for class_id in set(class_ids_2): + content_of_class = content_2[class_ids_2 == class_id] + stored_content_of_class = self._data_2.get(class_id, len0_like(content_2)) + self._data_2[class_id] = np.vstack( + (stored_content_of_class, content_of_class) + ) def __iter__( self, - ) -> Iterator[Tuple[int, Optional[npt.NDArray], Optional[npt.NDArray]]]: - class_ids = sorted( - set.union(set(self._data_1.keys()), set(self._data_2.keys())) - ) + ) -> Iterator[Tuple[int, npt.NDArray, npt.NDArray]]: + class_ids = sorted(set(self._data_1.keys()) | set(self._data_2.keys())) for class_id in class_ids: yield ( class_id, - self._data_1.get(class_id, None), - self._data_2.get(class_id, None), + self._data_1.get(class_id, self._make_empty()), + self._data_2.get(class_id, self._make_empty()), ) - def _get_content( - self, data: Union[npt.NDArray, Detections] - ) -> Optional[npt.NDArray]: + def _get_content(self, data: Union[npt.NDArray, Detections]) -> npt.NDArray: """Return boxes, masks or oriented bounding boxes from the data.""" if not isinstance(data, (Detections, np.ndarray)): raise ValueError( @@ -169,21 +162,24 @@ class InternalMetricDataStore: if self._metric_target == MetricTarget.BOXES: return data.xyxy if self._metric_target == MetricTarget.MASKS: - return data.mask + return ( + data.mask if data.mask is not None else np.zeros((0, 0, 0), dtype=bool) + ) if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: - obb = data.data.get(config.ORIENTED_BOX_COORDINATES, None) - if isinstance(obb, list): - obb = np.array(obb, dtype=np.float32) - return obb + obb = data.data.get( + config.ORIENTED_BOX_COORDINATES, np.zeros((0, 8), dtype=np.float32) + ) + return np.array(obb, dtype=np.float32) raise ValueError(f"Invalid metric target: {self._metric_target}") def _get_class_ids( self, data: Union[npt.NDArray, Detections] ) -> npt.NDArray[np.int_]: - if self._class_agnostic or isinstance(data, np.ndarray): - return np.array([CLASS_ID_NONE] * len(data), dtype=int) - assert isinstance(data, Detections) - if data.class_id is None: + if ( + self._class_agnostic + or isinstance(data, np.ndarray) + or data.class_id is None + ): return np.array([CLASS_ID_NONE] * len(data), dtype=int) return data.class_id @@ -197,12 +193,43 @@ class InternalMetricDataStore: ) def _validate_shape(self, data: npt.NDArray) -> None: - if self._datapoint_shape is None: - assert self._metric_target == MetricTarget.MASKS - self._datapoint_shape = data.shape[1:] - return - if data.shape[1:] != self._datapoint_shape: - raise ValueError( - f"Invalid data shape: {data.shape}." - f" Expected: (N, {self._datapoint_shape})" - ) + shape = data.shape + if self._metric_target == MetricTarget.BOXES: + if len(shape) != 2 or shape[1] != 4: + raise ValueError(f"Invalid xyxy shape: {shape}. Expected: (N, 4)") + elif self._metric_target == MetricTarget.MASKS: + if len(shape) != 3: + raise ValueError(f"Invalid mask shape: {shape}. Expected: (N, H, W)") + elif self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + if len(shape) != 2 or shape[1] != 8: + raise ValueError(f"Invalid obb shape: {shape}. Expected: (N, 8)") + else: + raise ValueError(f"Invalid metric target: {self._metric_target}") + + def _expand_mask_shape(self, data: npt.NDArray) -> npt.NDArray: + """Pad the stored and new data to the same shape.""" + if self._metric_target != MetricTarget.MASKS: + return data + + new_width = max(self._mask_shape[0], data.shape[1]) + new_height = max(self._mask_shape[1], data.shape[2]) + self._mask_shape = (new_width, new_height) + + data = pad_mask(data, self._mask_shape) + + for class_id, prev_data in self._data_1.items(): + self._data_1[class_id] = pad_mask(prev_data, self._mask_shape) + for class_id, prev_data in self._data_2.items(): + self._data_2[class_id] = pad_mask(prev_data, self._mask_shape) + + return data + + def _make_empty(self) -> npt.NDArray: + """Create an empty data object with the best-known shape for the target.""" + if self._metric_target == MetricTarget.BOXES: + return np.empty((0, 4), dtype=np.float32) + if self._metric_target == MetricTarget.MASKS: + return np.empty((0, *self._mask_shape), dtype=bool) + if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: + return np.empty((0, 8), dtype=np.float32) + raise ValueError(f"Invalid metric target: {self._metric_target}") diff --git a/supervision/metrics/intersection_over_union.py b/supervision/metrics/intersection_over_union.py index a8ed9d5a..9b3e4af6 100644 --- a/supervision/metrics/intersection_over_union.py +++ b/supervision/metrics/intersection_over_union.py @@ -1,4 +1,4 @@ -from typing import Dict, Union +from typing import Dict, List, Union import numpy as np import numpy.typing as npt @@ -33,27 +33,40 @@ class IntersectionOverUnion(Metric): def update( self, - data_1: Union[npt.NDArray, Detections], - data_2: Union[npt.NDArray, Detections], + data_1: Union[Detections, List[Detections]], + data_2: Union[Detections, List[Detections]], ) -> Self: """ Add data to the metric, without computing the result. - The arguments can be: - - * Boxes of shape `(N, 4)`, `float32`, - * Masks of shape `(N, H, W)`, `bool` - * Oriented bounding boxes of shape `(N, 8)`, `float32`. - * Detections object. - Args: - data_1 (Union[npt.NDArray, Detection]): The first set of data. - data_2 (Union[npt.NDArray, Detection]): The second set of data. + data_1 (Union[Detection, List[Detections]]): The first set of data. + data_2 (Union[Detection, List[Detections]]): The second set of data. Returns: Metric: The metric object itself. You can get the metric result by calling the `compute` method. """ + + if isinstance(data_1, list): + for d1 in data_1: + self.update(d1, Detections.empty()) + else: + self._update(data_1, Detections.empty()) + + if isinstance(data_2, list): + for d2 in data_2: + self.update(Detections.empty(), d2) + else: + self._update(Detections.empty(), data_2) + + return self + + def _update( + self, + data_1: Union[Detections], + data_2: Union[Detections], + ) -> Self: self._store.update(data_1, data_2) return self @@ -66,7 +79,6 @@ class IntersectionOverUnion(Metric): Dict[int, npt.NDArray[np.float32]]: A dictionary with class IDs as keys. If no class ID is provided, the key is the value CLASS_ID_NONE. """ - # TODO: cache computed result. ious = {} for class_id, array_1, array_2 in self._store: if self._metric_target == MetricTarget.BOXES: @@ -78,7 +90,7 @@ class IntersectionOverUnion(Metric): else: raise NotImplementedError( "Intersection over union is not implemented" - " for {self._metric_target}." + f" for {self._metric_target}." ) ious[class_id] = iou return ious diff --git a/supervision/metrics/utils.py b/supervision/metrics/utils.py new file mode 100644 index 00000000..737623ed --- /dev/null +++ b/supervision/metrics/utils.py @@ -0,0 +1,28 @@ +from typing import Tuple + +import numpy as np +import numpy.typing as npt + + +def pad_mask(mask: npt.NDArray, new_shape: Tuple[int, int]) -> npt.NDArray: + """Pad a mask to a new shape, inserting zeros on the right and bottom.""" + if len(mask.shape) != 3: + raise ValueError(f"Invalid mask shape: {mask.shape}. Expected: (N, H, W)") + + new_mask = np.pad( + mask, + ( + (0, 0), + (0, new_shape[0] - mask.shape[1]), + (0, new_shape[1] - mask.shape[2]), + ), + mode="constant", + constant_values=0, + ) + + return new_mask + + +def len0_like(data: npt.NDArray) -> npt.NDArray: + """Create an empty array with the same shape as input, but with 0 rows.""" + return np.empty((0, *data.shape[1:]), dtype=data.dtype) diff --git a/test/metrics/test_core.py b/test/metrics/test_core.py index e8f35b0b..b99e3633 100644 --- a/test/metrics/test_core.py +++ b/test/metrics/test_core.py @@ -24,6 +24,9 @@ def mock_xyxy(*box_index: int, box_width=10) -> npt.NDArray[np.float32]: For each index in `box_index`, a box is generated with the top-left corner at (i, i) and the bottom-right corner at (i + box_width, i + box_width). """ + if len(box_index) == 0: + return np.zeros((0, 4), dtype=np.float32) + box_list = [] for i in box_index: x0 = y0 = i @@ -112,49 +115,49 @@ def helper_test_store( ( mock_detections(1), mock_detections(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_detections(1), mock_xyxy(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_xyxy(1), mock_detections(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_xyxy(1), mock_xyxy(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_detections(), mock_detections(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_detections(), mock_xyxy(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_xyxy(), mock_detections(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_xyxy(), mock_xyxy(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), # More boxes @@ -165,7 +168,7 @@ def helper_test_store( ( CLASS_ID_NONE, mock_xyxy(1, 2), - None, + mock_xyxy(), ) ], DoesNotRaise(), @@ -192,7 +195,7 @@ def helper_test_store( ( mock_detections(1, 2, class_id=[1, 2]), mock_detections(), - [(CLASS_ID_NONE, mock_xyxy(1, 2), None)], + [(CLASS_ID_NONE, mock_xyxy(1, 2), mock_xyxy())], DoesNotRaise(), ), ( @@ -231,49 +234,49 @@ def test_store_boxes_class_agnostic( ( mock_detections(1), mock_detections(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_detections(1), mock_xyxy(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_xyxy(1), mock_detections(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_xyxy(1), mock_xyxy(), - [(CLASS_ID_NONE, mock_xyxy(1), None)], + [(CLASS_ID_NONE, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_detections(), mock_detections(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_detections(), mock_xyxy(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_xyxy(), mock_detections(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), ( mock_xyxy(), mock_xyxy(1), - [(CLASS_ID_NONE, None, mock_xyxy(1))], + [(CLASS_ID_NONE, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), # More boxes @@ -284,7 +287,7 @@ def test_store_boxes_class_agnostic( ( CLASS_ID_NONE, mock_xyxy(1, 2), - None, + mock_xyxy(), ) ], DoesNotRaise(), @@ -333,13 +336,13 @@ def test_store_boxes_by_class_regression( ( mock_detections(1, class_id=[1]), mock_detections(), - [(1, mock_xyxy(1), None)], + [(1, mock_xyxy(1), mock_xyxy())], DoesNotRaise(), ), ( mock_detections(), mock_detections(1, class_id=[1]), - [(1, None, mock_xyxy(1))], + [(1, mock_xyxy(), mock_xyxy(1))], DoesNotRaise(), ), # Multiple classes @@ -347,8 +350,8 @@ def test_store_boxes_by_class_regression( mock_detections(1, 2, class_id=[1, 2]), mock_detections(), [ - (1, mock_xyxy(1), None), - (2, mock_xyxy(2), None), + (1, mock_xyxy(1), mock_xyxy()), + (2, mock_xyxy(2), mock_xyxy()), ], DoesNotRaise(), ), @@ -356,13 +359,13 @@ def test_store_boxes_by_class_regression( mock_detections(1, 2, class_id=[1, 2]), mock_detections(3, 4, 5, class_id=[2, 3, 3]), [ - (1, mock_xyxy(1), None), + (1, mock_xyxy(1), mock_xyxy()), ( 2, mock_xyxy(2), mock_xyxy(3), ), - (3, None, mock_xyxy(4, 5)), + (3, mock_xyxy(), mock_xyxy(4, 5)), ], DoesNotRaise(), ), @@ -391,13 +394,13 @@ def test_store_boxes_by_class( ( [], mock_detections(), - [(1, mock_xyxy(), None)], + [(1, mock_xyxy(), mock_xyxy())], pytest.raises(ValueError), ), ( mock_detections(), [], - [(1, None, mock_xyxy())], + [(1, mock_xyxy(), mock_xyxy())], pytest.raises(ValueError), ), ], @@ -438,49 +441,49 @@ def test_store_boxes_invalid_args( ( mock_detections(1, with_mask=True), mock_detections(with_mask=True), - [(CLASS_ID_NONE, mock_mask(1), None)], + [(CLASS_ID_NONE, mock_mask(1), mock_mask())], DoesNotRaise(), ), ( mock_detections(1, with_mask=True), mock_mask(), - [(CLASS_ID_NONE, mock_mask(1), None)], + [(CLASS_ID_NONE, mock_mask(1), mock_mask())], DoesNotRaise(), ), ( mock_mask(1), mock_detections(with_mask=True), - [(CLASS_ID_NONE, mock_mask(1), None)], + [(CLASS_ID_NONE, mock_mask(1), mock_mask())], DoesNotRaise(), ), ( mock_mask(1), mock_mask(), - [(CLASS_ID_NONE, mock_mask(1), None)], + [(CLASS_ID_NONE, mock_mask(1), mock_mask())], DoesNotRaise(), ), ( mock_detections(with_mask=True), mock_detections(1, with_mask=True), - [(CLASS_ID_NONE, None, mock_mask(1))], + [(CLASS_ID_NONE, mock_mask(), mock_mask(1))], DoesNotRaise(), ), ( mock_detections(with_mask=True), mock_mask(1), - [(CLASS_ID_NONE, None, mock_mask(1))], + [(CLASS_ID_NONE, mock_mask(), mock_mask(1))], DoesNotRaise(), ), ( mock_mask(), mock_detections(1, with_mask=True), - [(CLASS_ID_NONE, None, mock_mask(1))], + [(CLASS_ID_NONE, mock_mask(), mock_mask(1))], DoesNotRaise(), ), ( mock_mask(), mock_mask(1), - [(CLASS_ID_NONE, None, mock_mask(1))], + [(CLASS_ID_NONE, mock_mask(), mock_mask(1))], DoesNotRaise(), ), # More masks @@ -491,7 +494,7 @@ def test_store_boxes_invalid_args( ( CLASS_ID_NONE, mock_mask(1, 2), - None, + mock_mask(), ) ], DoesNotRaise(), @@ -518,7 +521,7 @@ def test_store_boxes_invalid_args( ( mock_detections(1, 2, class_id=[1, 2], with_mask=True), mock_detections(with_mask=True), - [(CLASS_ID_NONE, mock_mask(1, 2), None)], + [(CLASS_ID_NONE, mock_mask(1, 2), mock_mask())], DoesNotRaise(), ), (