diff --git a/docs/metrics/detection.md b/docs/metrics/detection.md index 9e784464..feea7cce 100644 --- a/docs/metrics/detection.md +++ b/docs/metrics/detection.md @@ -6,3 +6,7 @@ ## ConfusionMatrix :::supervision.metrics.detection.ConfusionMatrix + +## MeanAveragePrecision + +:::supervision.metrics.detection.MeanAveragePrecision diff --git a/supervision/__init__.py b/supervision/__init__.py index ee2c7b31..8b8a11d4 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -30,12 +30,8 @@ from supervision.draw.color import Color, ColorPalette from supervision.draw.utils import draw_filled_rectangle, draw_polygon, draw_text from supervision.geometry.core import Point, Position, Rect from supervision.geometry.utils import get_polygon_center -from supervision.metrics.detection import ConfusionMatrix -from supervision.tracker.byte_tracker.byte_tracker import ( - ByteTrack, - detections2boxes, - match_detections_with_tracks, -) +from supervision.metrics.detection import ConfusionMatrix, MeanAveragePrecision +from supervision.tracker.byte_tracker.byte_tracker import ByteTrack from supervision.utils.file import list_files_with_extensions from supervision.utils.image import ImageSink, crop from supervision.utils.notebook import plot_image, plot_images_grid diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index ce7e0edb..6e47d747 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -31,7 +31,6 @@ from supervision.dataset.utils import ( train_test_split, ) from supervision.detection.core import Detections -from supervision.utils.file import list_files_with_extensions @dataclass @@ -197,7 +196,10 @@ class DetectionDataset(BaseDataset): @classmethod def from_pascal_voc( - cls, images_directory_path: str, annotations_directory_path: str + cls, + images_directory_path: str, + annotations_directory_path: str, + force_masks: bool = False, ) -> DetectionDataset: """ Creates a Dataset instance from PASCAL VOC formatted data. @@ -205,6 +207,7 @@ class DetectionDataset(BaseDataset): Args: images_directory_path (str): The path to the directory containing the images. annotations_directory_path (str): The path to the directory containing the PASCAL VOC XML annotations. + force_masks (bool, optional): If True, forces masks to be loaded for all annotations, regardless of whether they are present. Returns: DetectionDataset: A DetectionDataset instance containing the loaded images and annotations. @@ -222,7 +225,7 @@ class DetectionDataset(BaseDataset): >>> project = rf.workspace(WORKSPACE_ID).project(PROJECT_ID) >>> dataset = project.version(PROJECT_VERSION).download("voc") - >>> ds = sv.DetectionDataset.from_yolo( + >>> ds = sv.DetectionDataset.from_pascal_voc( ... images_directory_path=f"{dataset.location}/train/images", ... annotations_directory_path=f"{dataset.location}/train/labels" ... ) @@ -231,34 +234,13 @@ class DetectionDataset(BaseDataset): ['dog', 'person'] ``` """ - image_paths = list_files_with_extensions( - directory=images_directory_path, extensions=["jpg", "jpeg", "png"] - ) - annotation_paths = list_files_with_extensions( - directory=annotations_directory_path, extensions=["xml"] + + classes, images, annotations = load_pascal_voc_annotations( + images_directory_path=images_directory_path, + annotations_directory_path=annotations_directory_path, + force_masks=force_masks, ) - raw_annotations: List[Tuple[str, Detections, List[str]]] = [ - load_pascal_voc_annotations(annotation_path=str(annotation_path)) - for annotation_path in annotation_paths - ] - - classes = [] - for annotation in raw_annotations: - classes.extend(annotation[2]) - classes = list(set(classes)) - - for annotation in raw_annotations: - class_id = [classes.index(class_name) for class_name in annotation[2]] - annotation[1].class_id = np.array(class_id) - - images = { - image_path.name: cv2.imread(str(image_path)) for image_path in image_paths - } - - annotations = { - image_name: detections for image_name, detections, _ in raw_annotations - } return DetectionDataset(classes=classes, images=images, annotations=annotations) @classmethod diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index a3005eb9..4efb2917 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -1,12 +1,16 @@ -from typing import List, Optional, Tuple +import os +from pathlib import Path +from typing import Dict, List, Optional, Tuple from xml.dom.minidom import parseString from xml.etree.ElementTree import Element, SubElement, parse, tostring +import cv2 import numpy as np from supervision.dataset.utils import approximate_mask_with_polygons from supervision.detection.core import Detections -from supervision.detection.utils import polygon_to_xyxy +from supervision.detection.utils import polygon_to_mask, polygon_to_xyxy +from supervision.utils.file import list_files_with_extensions def object_to_pascal_voc( @@ -120,24 +124,91 @@ def detections_to_pascal_voc( def load_pascal_voc_annotations( - annotation_path: str, -) -> Tuple[str, Detections, List[str]]: + images_directory_path: str, + annotations_directory_path: str, + force_masks: bool = False, +) -> Tuple[List[str], Dict[str, np.ndarray], Dict[str, Detections]]: """ - Loads PASCAL VOC XML annotations and returns the image name, a Detections instance, and a list of class names. + Loads PASCAL VOC annotations and returns class names, images, and their corresponding detections. Args: - annotation_path (str): The path to the PASCAL VOC XML annotations file. + images_directory_path (str): The path to the directory containing the images. + annotations_directory_path (str): The path to the directory containing the PASCAL VOC annotation files. + force_masks (bool, optional): If True, forces masks to be loaded for all annotations, regardless of whether they are present. Returns: - Tuple[str, Detections, List[str]]: A tuple containing the image name, a Detections instance, and a list of class names of objects in the detections. + Tuple[List[str], Dict[str, np.ndarray], Dict[str, Detections]]: A tuple containing a list of class names, a dictionary with image names as keys and images as values, and a dictionary with image names as keys and corresponding Detections instances as values. """ - tree = parse(annotation_path) - root = tree.getroot() - image_name = root.find("filename").text + image_paths = list_files_with_extensions( + directory=images_directory_path, extensions=["jpg", "jpeg", "png"] + ) + classes = [] + images = {} + annotations = {} + + for image_path in image_paths: + image_name = Path(image_path).stem + image = cv2.imread(str(image_path)) + + annotation_path = os.path.join(annotations_directory_path, f"{image_name}.xml") + if not os.path.exists(annotation_path): + images[image_path.name] = image + annotations[image_path.name] = Detections.empty() + continue + + tree = parse(annotation_path) + root = tree.getroot() + + resolution_wh = (image.shape[1], image.shape[0]) + annotation, classes = detections_from_xml_obj( + root, classes, resolution_wh, force_masks + ) + + images[image_path.name] = image + annotations[image_path.name] = annotation + + return classes, images, annotations + + +def detections_from_xml_obj( + root: Element, classes: List[str], resolution_wh, force_masks: bool = False +) -> Tuple[Detections, List[str]]: + """ + Converts an XML object in Pascal VOC format to a Detections object. + Expected XML format: + + ... + + dog + + 48 + 240 + 195 + 371 + + + 48 + 240 + 195 + 240 + 195 + 371 + 48 + 371 + + + + + Returns: + Tuple[Detections, List[str]]: A tuple containing a Detections object and an updated list of class names, extended with the class names from the XML object. + """ xyxy = [] class_names = [] + masks = [] + with_masks = False + extended_classes = classes[:] for obj in root.findall("object"): class_name = obj.find("name").text class_names.append(class_name) @@ -150,7 +221,40 @@ def load_pascal_voc_annotations( xyxy.append([x1, y1, x2, y2]) - xyxy = np.array(xyxy) - detections = Detections(xyxy=xyxy) + with_masks = obj.find("polygon") is not None + with_masks = force_masks if force_masks else with_masks - return image_name, detections, class_names + for polygon in obj.findall("polygon"): + polygon_points = parse_polygon_points(polygon) + + mask_from_polygon = polygon_to_mask( + polygon=np.array(polygon_points), + resolution_wh=resolution_wh, + ) + masks.append(mask_from_polygon) + + xyxy = np.array(xyxy) if len(xyxy) > 0 else np.empty((0, 4)) + for k in set(class_names): + if k not in extended_classes: + extended_classes.append(k) + class_id = np.array( + [extended_classes.index(class_name) for class_name in class_names] + ) + + if with_masks: + annotation = Detections( + xyxy=xyxy, mask=np.array(masks).astype(bool), class_id=class_id + ) + else: + annotation = Detections(xyxy=xyxy, class_id=class_id) + return annotation, extended_classes + + +def parse_polygon_points(polygon: Element) -> List[List[int]]: + polygon_points = [] + coords = polygon.findall(".//*") + for i in range(0, len(coords), 2): + x = int(coords[i].text) + y = int(coords[i + 1].text) + polygon_points.append([x, y]) + return polygon_points diff --git a/supervision/dataset/formats/yolo.py b/supervision/dataset/formats/yolo.py index da070a0f..0d41f045 100644 --- a/supervision/dataset/formats/yolo.py +++ b/supervision/dataset/formats/yolo.py @@ -126,9 +126,6 @@ def load_yolo_annotations( image_paths = list_files_with_extensions( directory=images_directory_path, extensions=["jpg", "jpeg", "png"] ) - annotation_paths = list_files_with_extensions( - directory=annotations_directory_path, extensions=["txt"] - ) classes = _extract_class_names(file_path=data_yaml_path) images = {} diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 5b4e095b..825d00c7 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -7,12 +7,13 @@ import cv2 import numpy as np from supervision.detection.utils import ( - extract_yolov8_masks, + extract_ultralytics_masks, non_max_suppression, process_roboflow_result, xywh_to_xyxy, ) from supervision.geometry.core import Position +from supervision.utils.internal import deprecated def _validate_xyxy(xyxy: Any, n: int) -> None: @@ -26,7 +27,7 @@ def _validate_mask(mask: Any, n: int) -> None: isinstance(mask, np.ndarray) and len(mask.shape) == 3 and mask.shape[0] == n ) if not is_valid: - raise ValueError("mask must be 3d np.ndarray with (n, W, H) shape") + raise ValueError("mask must be 3d np.ndarray with (n, H, W) shape") def _validate_class_id(class_id: Any, n: int) -> None: @@ -59,7 +60,7 @@ class Detections: Data class containing information about the detections in a video frame. Attributes: xyxy (np.ndarray): An array of shape `(n, 4)` containing the bounding boxes coordinates in format `[x1, y1, x2, y2]` - mask: (Optional[np.ndarray]): An array of shape `(n, W, H)` containing the segmentation masks. + mask: (Optional[np.ndarray]): An array of shape `(n, H, W)` containing the segmentation masks. confidence (Optional[np.ndarray]): An array of shape `(n,)` containing the confidence scores of the detections. class_id (Optional[np.ndarray]): An array of shape `(n,)` containing the class ids of the detections. tracker_id (Optional[np.ndarray]): An array of shape `(n,)` containing the tracker ids of the detections. @@ -170,6 +171,9 @@ class Detections: ) @classmethod + @deprecated( + "This method is deprecated and removed in 0.15.0 release. Use sv.Detections.from_ultralytics() instead." + ) def from_yolov8(cls, yolov8_results) -> Detections: """ Creates a Detections instance from a [YOLOv8](https://github.com/ultralytics/ultralytics) inference result. @@ -196,7 +200,42 @@ class Detections: xyxy=yolov8_results.boxes.xyxy.cpu().numpy(), confidence=yolov8_results.boxes.conf.cpu().numpy(), class_id=yolov8_results.boxes.cls.cpu().numpy().astype(int), - mask=extract_yolov8_masks(yolov8_results), + mask=extract_ultralytics_masks(yolov8_results), + ) + + @classmethod + def from_ultralytics(cls, ultralytics_results) -> Detections: + """ + Creates a Detections instance from a [YOLOv8](https://github.com/ultralytics/ultralytics) inference result. + + Args: + yolov8_results (ultralytics.yolo.engine.results.Results): The output Results instance from YOLOv8 + + Returns: + Detections: A new Detections object. + + Example: + ```python + >>> import cv2 + >>> from ultralytics import YOLO, FastSAM, SAM, RTDETR + >>> import supervision as sv + + >>> image = cv2.imread(SOURCE_IMAGE_PATH) + >>> model = YOLO('yolov8s.pt') + >>> model = SAM('sam_b.pt') + >>> model = SAM('mobile_sam.pt') + >>> model = FastSAM('FastSAM-s.pt') + >>> model = RTDETR('rtdetr-l.pt') + + >>> result = model(image)[0] + >>> detections = sv.Detections.from_ultralytics(result) + ``` + """ + return cls( + xyxy=ultralytics_results.boxes.xyxy.cpu().numpy(), + confidence=ultralytics_results.boxes.conf.cpu().numpy(), + class_id=ultralytics_results.boxes.cls.cpu().numpy().astype(int), + mask=extract_ultralytics_masks(ultralytics_results), ) @classmethod diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 63206ddc..71bd019c 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -17,7 +17,7 @@ def polygon_to_mask(polygon: np.ndarray, resolution_wh: Tuple[int, int]) -> np.n np.ndarray: The generated 2D mask, where the polygon is marked with `1`'s and the rest is filled with `0`'s. """ width, height = resolution_wh - mask = np.zeros((height, width), dtype=np.uint8) + mask = np.zeros((height, width)) cv2.fillPoly(mask, [polygon], color=1) return mask @@ -260,7 +260,7 @@ def approximate_polygon( return np.squeeze(approximated_points, axis=1) -def extract_yolov8_masks(yolov8_results) -> Optional[np.ndarray]: +def extract_ultralytics_masks(yolov8_results) -> Optional[np.ndarray]: if not yolov8_results.masks: return None @@ -288,7 +288,10 @@ def extract_yolov8_masks(yolov8_results) -> Optional[np.ndarray]: for i in range(masks.shape[0]): mask = masks[i] mask = mask[top:bottom, left:right] - mask = cv2.resize(mask, (orig_shape[1], orig_shape[0])) + + if mask.shape != orig_shape: + mask = cv2.resize(mask, (orig_shape[1], orig_shape[0])) + mask_maps.append(mask) return np.asarray(mask_maps, dtype=bool) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 5b5f710a..d126f82e 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -12,6 +12,59 @@ from supervision.detection.core import Detections from supervision.detection.utils import box_iou_batch +def detections_to_tensor( + detections: Detections, with_confidence: bool = False +) -> np.ndarray: + """ + Convert Supervision Detections to numpy tensors for further computation + Args: + detections (sv.Detections): Detections/Targets in the format of sv.Detections + with_confidence (bool): Whether to include confidence in the tensor + Returns: + (np.ndarray): Detections as numpy tensors as in (xyxy, class_id, confidence) order + """ + if detections.class_id is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with class_id" + ) + + arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + + if with_confidence: + if detections.confidence is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with confidence" + ) + arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + + return np.concatenate(arrays_to_concat, axis=1) + + +def validate_input_tensors(predictions: List[np.ndarray], targets: List[np.ndarray]): + """ + Checks for shape consistency of input tensors. + """ + if len(predictions) != len(targets): + raise ValueError( + f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." + ) + if len(predictions) > 0: + if not isinstance(predictions[0], np.ndarray) or not isinstance( + targets[0], np.ndarray + ): + raise ValueError( + f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." + ) + if predictions[0].shape[1] != 6: + raise ValueError( + f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." + ) + if targets[0].shape[1] != 5: + raise ValueError( + f"Targets must have shape (N, 5). Got {targets[0].shape} instead." + ) + + @dataclass class ConfusionMatrix: """ @@ -85,11 +138,9 @@ class ConfusionMatrix: target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - ConfusionMatrix.detections_to_tensor(prediction, with_confidence=True) - ) - target_tensors.append( - ConfusionMatrix.detections_to_tensor(target, with_confidence=False) + detections_to_tensor(prediction, with_confidence=True) ) + target_tensors.append(detections_to_tensor(target, with_confidence=False)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, @@ -98,26 +149,6 @@ class ConfusionMatrix: iou_threshold=iou_threshold, ) - @staticmethod - def detections_to_tensor( - detections: Detections, with_confidence: bool = False - ) -> np.ndarray: - if detections.class_id is None: - raise ValueError( - "ConfusionMatrix can only be calculated for Detections with class_id" - ) - - arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] - - if with_confidence: - if detections.confidence is None: - raise ValueError( - "ConfusionMatrix can only be calculated for Detections with confidence" - ) - arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) - - return np.concatenate(arrays_to_concat, axis=1) - @classmethod def from_tensors( cls, @@ -184,7 +215,7 @@ class ConfusionMatrix: ]) ``` """ - cls._validate_input_tensors(predictions, targets) + validate_input_tensors(predictions, targets) num_classes = len(classes) matrix = np.zeros((num_classes + 1, num_classes + 1)) @@ -203,33 +234,6 @@ class ConfusionMatrix: iou_threshold=iou_threshold, ) - @classmethod - def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] - ): - """ - Checks for shape consistency of input tensors. - """ - if len(predictions) != len(targets): - raise ValueError( - f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." - ) - if len(predictions) > 0: - if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray - ): - raise ValueError( - f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." - ) - if predictions[0].shape[1] != 6: - raise ValueError( - f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." - ) - if targets[0].shape[1] != 5: - raise ValueError( - f"Targets must have shape (N, 5). Got {targets[0].shape} instead." - ) - @staticmethod def evaluate_detection_batch( predictions: np.ndarray, @@ -319,7 +323,7 @@ class ConfusionMatrix: iou_threshold: float = 0.5, ) -> ConfusionMatrix: """ - Create confusion matrix from dataset and callback function. + Calculate confusion matrix from dataset and callback function. Args: dataset (DetectionDataset): Object detection dataset used for evaluation. @@ -456,3 +460,331 @@ class ConfusionMatrix: save_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True ) return fig + + +@dataclass(frozen=True) +class MeanAveragePrecision: + """ + Mean Average Precision for object detection tasks. + + Attributes: + map (float): mAP value. + map50 (float): mAP value at IoU `threshold = 0.5`. + map75 (float): mAP value at IoU `threshold = 0.75`. + per_class_ap (np.ndarray): values for every classes. + """ + + map: float + map50: float + map75: float + per_class_ap: np.ndarray + + @classmethod + def from_detections( + cls, + predictions: List[Detections], + targets: List[Detections], + ) -> MeanAveragePrecision: + """ + Calculate mean average precision based on predicted and ground-truth detections. + + Args: + targets (List[Detections]): Detections objects from ground-truth. + predictions (List[Detections]): Detections objects predicted by the model. + Returns: + MeanAveragePrecision: New instance of ConfusionMatrix. + + Example: + ```python + >>> import supervision as sv + + >>> targets = [ + ... sv.Detections(...), + ... sv.Detections(...) + ... ] + + >>> predictions = [ + ... sv.Detections(...), + ... sv.Detections(...) + ... ] + + >>> mean_average_precision = sv.MeanAveragePrecision.from_detections( + ... predictions=predictions, + ... targets=target, + ... ) + + >>> mean_average_precison.map + 0.2899 + ``` + """ + prediction_tensors = [] + target_tensors = [] + for prediction, target in zip(predictions, targets): + prediction_tensors.append( + detections_to_tensor(prediction, with_confidence=True) + ) + target_tensors.append(detections_to_tensor(target, with_confidence=False)) + return cls.from_tensors( + predictions=prediction_tensors, + targets=target_tensors, + ) + + @classmethod + def benchmark( + cls, + dataset: DetectionDataset, + callback: Callable[[np.ndarray], Detections], + ) -> MeanAveragePrecision: + """ + Calculate mean average precision from dataset and callback function. + + Args: + dataset (DetectionDataset): Object detection dataset used for evaluation. + callback (Callable[[np.ndarray], Detections]): Function that takes an image as input and returns Detections object. + Returns: + MeanAveragePrecision: New instance of MeanAveragePrecision. + + Example: + ```python + >>> import supervision as sv + >>> from ultralytics import YOLO + + >>> dataset = sv.DetectionDataset.from_yolo(...) + + >>> model = YOLO(...) + >>> def callback(image: np.ndarray) -> sv.Detections: + ... result = model(image)[0] + ... return sv.Detections.from_yolov8(result) + + >>> mean_average_precision = sv.MeanAveragePrecision.benchmark( + ... dataset = dataset, + ... callback = callback + ... ) + + >>> mean_average_precision.map + 0.433 + ``` + """ + predictions, targets = [], [] + for img_name, img in dataset.images.items(): + predictions_batch = callback(img) + predictions.append(predictions_batch) + targets_batch = dataset.annotations[img_name] + targets.append(targets_batch) + return cls.from_detections( + predictions=predictions, + targets=targets, + ) + + @classmethod + def from_tensors( + cls, + predictions: List[np.ndarray], + targets: List[np.ndarray], + ) -> MeanAveragePrecision: + """ + Calculate Mean Average Precision based on predicted and ground-truth detections at different threshold. + + Args: + predictions (List[np.ndarray]): Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. + targets (List[np.ndarray]): Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. + Returns: + MeanAveragePrecision: New instance of MeanAveragePrecision. + + Example: + ```python + >>> import supervision as sv + + >>> targets = ( + ... [ + ... array( + ... [ + ... [0.0, 0.0, 3.0, 3.0, 1], + ... [2.0, 2.0, 5.0, 5.0, 1], + ... [6.0, 1.0, 8.0, 3.0, 2], + ... ] + ... ), + ... array([1.0, 1.0, 2.0, 2.0, 2]), + ... ] + ... ) + + >>> predictions = [ + ... array( + ... [ + ... [0.0, 0.0, 3.0, 3.0, 1, 0.9], + ... [0.1, 0.1, 3.0, 3.0, 0, 0.9], + ... [6.0, 1.0, 8.0, 3.0, 1, 0.8], + ... [1.0, 6.0, 2.0, 7.0, 1, 0.8], + ... ] + ... ), + ... array([[1.0, 1.0, 2.0, 2.0, 2, 0.8]]) + ... ] + + >>> mean_average_precison = sv.MeanAveragePrecision.from_tensors( + ... predictions=predictions, + ... targets=targets, + ... ) + + >>> mean_average_precison.map + 0.2899 + ``` + """ + validate_input_tensors(predictions, targets) + map, map50, map75 = 0, 0, 0 + + class_index = 4 + conf_index = 5 + + stats, average_precisions = [], [] + iou_levels = np.linspace(0.5, 0.95, 10) + num_ious = iou_levels.size + + for true_batch, detection_batch in zip(targets, predictions): + nl, npr = ( + true_batch.shape[0], + detection_batch.shape[0], + ) + correct = np.zeros((npr, num_ious), dtype=bool) + + if npr == 0: + if nl: + stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) + continue + if nl: + correct = MeanAveragePrecision._match_detection_batch( + predictions=detection_batch, + targets=true_batch, + iou_levels=iou_levels, + ) + stats.append( + ( + correct, + detection_batch[:, conf_index], + detection_batch[:, class_index], + true_batch[:, class_index], + ) + ) + + stats = [np.concatenate(x, 0) for x in zip(*stats)] + + if len(stats) and stats[0].any(): + average_precisions = cls._average_precisions_per_class(*stats) + ap50, ap75, average_precisions = ( + average_precisions[:, 0], + average_precisions[:, 5], + average_precisions.mean(1), + ) + map50, map75, map = ap50.mean(), ap75.mean(), average_precisions.mean() + + return cls(map=map, map50=map50, map75=map75, per_class_ap=average_precisions) + + @staticmethod + def _match_detection_batch( + predictions: np.ndarray, targets: np.ndarray, iou_levels: np.ndarray + ) -> np.ndarray: + """ + Args: + predictions (np.ndarray): batch prediction + targets (np.ndarray): batch target labels + iou_levels (np.ndarray): iou levels array contains different iou levels + + Returns: + (np.ndarray): matched prediction with target lebels result + """ + correct = np.zeros((predictions.shape[0], iou_levels.shape[0])).astype(bool) + iou = box_iou_batch(targets[:, :4], predictions[:, :4]) + + correct_class = targets[:, 4:5] == predictions[:, 4] + + for i in range(len(iou_levels)): + x = np.where((iou >= iou_levels[i]) & correct_class) + + if x[0].shape[0]: + _X1 = np.concatenate( + [np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1 + ) + _x2 = iou[x[0], x[1]][:, None] + matches = np.concatenate([_X1, _x2], axis=1) + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + correct[matches[:, 1].astype(int), i] = True + return correct + + @staticmethod + def compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float: + """ + Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves. + + Args: + recall (np.ndarray): The recall curve. + precision (np.ndarray): The precision curve. + + Returns: + float: Average precision. + """ + extended_recall = np.concatenate(([0.0], recall, [1.0])) + extended_precision = np.concatenate(([1.0], precision, [0.0])) + max_accumulated_precision = np.flip( + np.maximum.accumulate(np.flip(extended_precision)) + ) + interpolated_recall_levels = np.linspace(0, 1, 101) + interpolated_precision = np.interp( + interpolated_recall_levels, extended_recall, max_accumulated_precision + ) + average_precision = np.trapz(interpolated_precision, interpolated_recall_levels) + return average_precision + + @staticmethod + def _average_precisions_per_class( + matches: np.ndarray, + prediction_confidence: np.ndarray, + prediction_class_ids: np.ndarray, + true_batch_class_ids: np.ndarray, + eps: float = 1e-16, + ) -> np.ndarray: + """ + Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + Args: + matches (np.ndarray): True positives (nparray, nx1 or nx10). + prediction_confidence (np.ndarray): Objectness value from 0-1 (nparray). + prediction_class_ids (np.ndarray): Predicted object classes (nparray). + true_batch_class_ids (np.ndarray): True object classes (nparray). + Returns: + (np.ndarray): Average precision for different iou level array + """ + sorted_confidences = np.argsort(-prediction_confidence) + matches = matches[sorted_confidences] + prediction_class_ids = prediction_class_ids[sorted_confidences] + + # Find unique classes + unique_classes, class_counts = np.unique( + true_batch_class_ids, return_counts=True + ) + num_classes = unique_classes.shape[0] # number of classes, number of detections + + average_precisions = np.zeros((num_classes, matches.shape[1])) + for ci, c in enumerate(unique_classes): + valid = prediction_class_ids == c + num_targets = class_counts[ci] # number of labels + num_predictions = valid.sum() # number of predictions + if num_predictions == 0 or num_targets == 0: + continue + + fp_pool = (1 - matches[valid]).cumsum(0) + tp_pool = matches[valid].cumsum(0) + + recall = tp_pool / (num_targets + eps) + precision = tp_pool / (tp_pool + fp_pool) + + for j in range(matches.shape[1]): + average_precisions[ + ci, j + ] = MeanAveragePrecision.compute_average_precision( + recall[:, j], precision[:, j] + ) + + return average_precisions diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py new file mode 100644 index 00000000..fa3f1124 --- /dev/null +++ b/test/dataset/formats/test_pascal_voc.py @@ -0,0 +1,147 @@ +import xml.etree.ElementTree as ET +from contextlib import ExitStack as DoesNotRaise +from test.utils import mock_detections +from typing import List, Optional + +import numpy as np +import pytest + +from supervision.dataset.formats.pascal_voc import ( + detections_from_xml_obj, + object_to_pascal_voc, + parse_polygon_points, +) + + +def are_xml_elements_equal(elem1, elem2): + if ( + elem1.tag != elem2.tag + or elem1.attrib != elem2.attrib + or elem1.text != elem2.text + or len(elem1) != len(elem2) + ): + return False + + for child1, child2 in zip(elem1, elem2): + if not are_xml_elements_equal(child1, child2): + return False + + return True + + +@pytest.mark.parametrize( + "xyxy, name, polygon, expected_result, exception", + [ + ( + [0, 0, 10, 10], + "test", + None, + ET.fromstring( + """test001010""" + ), + DoesNotRaise(), + ), + ( + [0, 0, 10, 10], + "test", + [[0, 0], [10, 0], [10, 10], [0, 10]], + ET.fromstring( + """test001010001001010010""" + ), + DoesNotRaise(), + ), + ], +) +def test_object_to_pascal_voc( + xyxy: np.ndarray, + name: str, + polygon: Optional[np.ndarray], + expected_result, + exception: Exception, +): + with exception: + result = object_to_pascal_voc(xyxy=xyxy, name=name, polygon=polygon) + assert are_xml_elements_equal(result, expected_result) + + +@pytest.mark.parametrize( + "polygon_element, expected_result, exception", + [ + ( + ET.fromstring( + """001001010010""" + ), + [[0, 0], [10, 0], [10, 10], [0, 10]], + DoesNotRaise(), + ) + ], +) +def test_parse_polygon_points( + polygon_element, + expected_result: List[list], + exception, +): + with exception: + result = parse_polygon_points(polygon_element) + assert result == expected_result + + +ONE_CLASS_N_BBOX = """test001010test10102020""" + + +ONE_CLASS_ONE_BBOX = """test001010""" + + +N_CLASS_N_BBOX = """test001010test20303040test210102020""" + +NO_DETECTIONS = """""" + + +@pytest.mark.parametrize( + "xml_string, classes, resolution_wh, force_masks, expected_result, exception", + [ + ( + ONE_CLASS_ONE_BBOX, + ["test"], + (100, 100), + False, + mock_detections(np.array([[0, 0, 10, 10]]), None, [0]), + DoesNotRaise(), + ), + ( + ONE_CLASS_N_BBOX, + ["test"], + (100, 100), + False, + mock_detections(np.array([[0, 0, 10, 10], [10, 10, 20, 20]]), None, [0, 0]), + DoesNotRaise(), + ), + ( + N_CLASS_N_BBOX, + ["test", "test2"], + (100, 100), + False, + mock_detections( + np.array([[0, 0, 10, 10], [20, 30, 30, 40], [10, 10, 20, 20]]), + None, + [0, 0, 1], + ), + DoesNotRaise(), + ), + ( + NO_DETECTIONS, + [], + (100, 100), + False, + mock_detections(np.empty((0, 4)), None, []), + DoesNotRaise(), + ), + ], +) +def test_detections_from_xml_obj( + xml_string, classes, resolution_wh, force_masks, expected_result, exception +): + with exception: + root = ET.fromstring(xml_string) + result, _ = detections_from_xml_obj(root, classes, resolution_wh, force_masks) + assert result == expected_result diff --git a/test/metrics/test_detection.py b/test/metrics/test_detection.py index 0ecab3f0..e0a5f2ce 100644 --- a/test/metrics/test_detection.py +++ b/test/metrics/test_detection.py @@ -5,8 +5,8 @@ import numpy as np import pytest from supervision.detection.core import Detections -from supervision.metrics.detection import ConfusionMatrix -from test.utils import mock_detections +from supervision.metrics.detection import ConfusionMatrix, detections_to_tensor, MeanAveragePrecision +from test.utils import mock_detections, assert_almost_equal CLASSES = np.arange(80) NUM_CLASSES = len(CLASSES) @@ -167,7 +167,7 @@ def test_detections_to_tensor( exception: Exception ): with exception: - result = ConfusionMatrix.detections_to_tensor( + result = detections_to_tensor( detections=detections, with_confidence=with_confidence ) @@ -400,3 +400,43 @@ def test_drop_extra_matches( result = ConfusionMatrix._drop_extra_matches(matches) assert np.array_equal(result, expected_result) + + +@pytest.mark.parametrize( + 'recall, precision, expected_result, exception', + [ + ( + np.array([1.0]), + np.array([1.0]), + 1.0, + DoesNotRaise() + ), # perfect recall and precision + ( + np.array([0.0]), + np.array([0.0]), + 0.0, + DoesNotRaise() + ), # no recall and precision + ( + np.array([0.0, 0.2, 0.2, 0.8, 0.8, 1.0]), + np.array([0.7, 0.8, 0.4, 0.5, 0.1, 0.2]), + 0.5, + DoesNotRaise() + ), + ( + np.array([0.0, 0.5, 0.5, 1.0]), + np.array([0.75, 0.75, 0.75, 0.75]), + 0.75, + DoesNotRaise() + ) + ] +) +def test_compute_average_precision( + recall: np.ndarray, + precision: np.ndarray, + expected_result: float, + exception: Exception +) -> None: + with exception: + result = MeanAveragePrecision.compute_average_precision(recall=recall, precision=precision) + assert_almost_equal(result, expected_result, tolerance=0.01) diff --git a/test/utils.py b/test/utils.py index 20c294e5..a1d6907e 100644 --- a/test/utils.py +++ b/test/utils.py @@ -21,3 +21,7 @@ def mock_detections( if tracker_id is None else np.array(tracker_id, dtype=int), ) + + +def assert_almost_equal(actual, expected, tolerance=1e-5): + assert abs(actual - expected) < tolerance, f"Expected {expected}, but got {actual}."