From e882c398b78322afafafb7b7e743ab90c0734a62 Mon Sep 17 00:00:00 2001 From: hd Date: Sat, 22 Jul 2023 16:10:31 +0200 Subject: [PATCH 01/54] Added method for complete ultralytics model usage --- supervision/detection/core.py | 41 ++++++++++++++++++++++++++++++++-- supervision/detection/utils.py | 7 ++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 9aab3630..e8cb72c9 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -7,11 +7,12 @@ import cv2 import numpy as np from supervision.detection.utils import ( - extract_yolov8_masks, + extract_ultralytics_masks, non_max_suppression, process_roboflow_result, xywh_to_xyxy, ) +from supervision.utils.internal import deprecated from supervision.geometry.core import Position @@ -170,6 +171,7 @@ class Detections: ) @classmethod + @deprecated("Please use sv.Detections.from_ultralytics() API for future usage. This method is deprecated and removed in future release") def from_yolov8(cls, yolov8_results) -> Detections: """ Creates a Detections instance from a [YOLOv8](https://github.com/ultralytics/ultralytics) inference result. @@ -196,7 +198,42 @@ class Detections: xyxy=yolov8_results.boxes.xyxy.cpu().numpy(), confidence=yolov8_results.boxes.conf.cpu().numpy(), class_id=yolov8_results.boxes.cls.cpu().numpy().astype(int), - mask=extract_yolov8_masks(yolov8_results), + mask=extract_ultralytics_masks(yolov8_results), + ) + + @classmethod + def from_ultralytics(cls, ultralytics_results) -> Detections: + """ + Creates a Detections instance from a [YOLOv8](https://github.com/ultralytics/ultralytics) inference result. + + Args: + yolov8_results (ultralytics.yolo.engine.results.Results): The output Results instance from YOLOv8 + + Returns: + Detections: A new Detections object. + + Example: + ```python + >>> import cv2 + >>> from ultralytics import YOLO, FastSAM, SAM, RTDETR + >>> import supervision as sv + + >>> image = cv2.imread(SOURCE_IMAGE_PATH) + >>> model = YOLO('yolov8s.pt') + >>> model = SAM('sam_b.pt') + >>> model = SAM('mobile_sam.pt') + >>> model = FastSAM('FastSAM-s.pt') + >>> model = RTDETR('FastSAM-s.pt') + + >>> result = model(image)[0] + >>> detections = sv.Detections.from_ultralytics(result) + ``` + """ + return cls( + xyxy=ultralytics_results.boxes.xyxy.cpu().numpy(), + confidence=ultralytics_results.boxes.conf.cpu().numpy(), + class_id=ultralytics_results.boxes.cls.cpu().numpy().astype(int), + mask=extract_ultralytics_masks(ultralytics_results), ) @classmethod diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 63206ddc..49d59188 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -260,7 +260,7 @@ def approximate_polygon( return np.squeeze(approximated_points, axis=1) -def extract_yolov8_masks(yolov8_results) -> Optional[np.ndarray]: +def extract_ultralytics_masks(yolov8_results) -> Optional[np.ndarray]: if not yolov8_results.masks: return None @@ -288,7 +288,10 @@ def extract_yolov8_masks(yolov8_results) -> Optional[np.ndarray]: for i in range(masks.shape[0]): mask = masks[i] mask = mask[top:bottom, left:right] - mask = cv2.resize(mask, (orig_shape[1], orig_shape[0])) + + if mask.shape != orig_shape: + mask = cv2.resize(mask, (orig_shape[1], orig_shape[0])) + mask_maps.append(mask) return np.asarray(mask_maps, dtype=bool) From 3c596181522e51d5050fd39181938193c77373b1 Mon Sep 17 00:00:00 2001 From: hd Date: Sat, 22 Jul 2023 16:12:31 +0200 Subject: [PATCH 02/54] ready for review --- supervision/dataset/core.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index a081d2fb..7167171a 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -511,18 +511,6 @@ class DetectionDataset(BaseDataset): classes=merged_classes, images=merged_images, annotations=merged_annotations ) - def add_class_names(self, class_names: List[str]): - self.classes = class_names - - def add_instance(self, filename: str, image: np.ndarray, detections: Detections): - if filename not in self.annotations.keys(): - self.images[filename] = image - self.annotations[filename] = detections - - @classmethod - def emty(cls): - return cls(classes=[], images={}, annotations={}) - @dataclass class ClassificationDataset(BaseDataset): From 5edd58d6de401a16fd36c371589ce18ed63db7f3 Mon Sep 17 00:00:00 2001 From: hd Date: Sat, 22 Jul 2023 20:45:41 +0200 Subject: [PATCH 03/54] ready for review --- supervision/detection/core.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index e8cb72c9..033a41e5 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -12,8 +12,8 @@ from supervision.detection.utils import ( process_roboflow_result, xywh_to_xyxy, ) -from supervision.utils.internal import deprecated from supervision.geometry.core import Position +from supervision.utils.internal import deprecated def _validate_xyxy(xyxy: Any, n: int) -> None: @@ -171,7 +171,9 @@ class Detections: ) @classmethod - @deprecated("Please use sv.Detections.from_ultralytics() API for future usage. This method is deprecated and removed in future release") + @deprecated( + "Please use sv.Detections.from_ultralytics() API for future usage. This method is deprecated and removed in future release" + ) def from_yolov8(cls, yolov8_results) -> Detections: """ Creates a Detections instance from a [YOLOv8](https://github.com/ultralytics/ultralytics) inference result. @@ -223,7 +225,7 @@ class Detections: >>> model = SAM('sam_b.pt') >>> model = SAM('mobile_sam.pt') >>> model = FastSAM('FastSAM-s.pt') - >>> model = RTDETR('FastSAM-s.pt') + >>> model = RTDETR('rtdetr-l.pt') >>> result = model(image)[0] >>> detections = sv.Detections.from_ultralytics(result) From 6def59444dc57f3f1b4f53c3e4f59aa49570d247 Mon Sep 17 00:00:00 2001 From: hd Date: Sun, 23 Jul 2023 13:27:21 +0200 Subject: [PATCH 04/54] initial commit --- supervision/__init__.py | 2 +- supervision/metrics/detection.py | 265 +++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 1 deletion(-) diff --git a/supervision/__init__.py b/supervision/__init__.py index 9c0252fb..b817d113 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -30,7 +30,7 @@ from supervision.draw.color import Color, ColorPalette from supervision.draw.utils import draw_filled_rectangle, draw_polygon, draw_text from supervision.geometry.core import Point, Position, Rect from supervision.geometry.utils import get_polygon_center -from supervision.metrics.detection import ConfusionMatrix +from supervision.metrics.detection import ConfusionMatrix, AveragePrecision, MeanAveragePrecision from supervision.utils.file import list_files_with_extensions from supervision.utils.image import ImageSink, crop from supervision.utils.notebook import plot_image, plot_images_grid diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index e7234973..10704525 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -456,3 +456,268 @@ class ConfusionMatrix: save_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True ) return fig + + +@dataclass +class AveragePrecision: + + value: float + recall_values: np.ndarray + precision_values: np.ndarray + class_idx: Optional[int] = None + iou_threshold: Optional[float] = None + + @classmethod + def from_detections( + cls, + predictions: List[Detections], + targets: List[Detections], + class_idx: int, + iou_threshold: float = 0.5, + ) -> AveragePrecision: + + prediction_tensors = [] + target_tensors = [] + for prediction, target in zip(predictions, targets): + prediction_tensors.append( + cls.detections_to_tensor(prediction, with_confidence=True) + ) + target_tensors.append( + cls.detections_to_tensor(target, with_confidence=False) + ) + return cls.from_tensors( + predictions=prediction_tensors, + targets=target_tensors, + class_idx=class_idx, + iou_threshold=iou_threshold, + ) + + @classmethod + def detections_to_tensor( + cls, detections: Detections, with_confidence: bool = False + ) -> np.ndarray: + if detections.class_id is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with class_id" + ) + + arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + + if with_confidence: + if detections.confidence is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with confidence" + ) + arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + + return np.concatenate(arrays_to_concat, axis=1) + + @classmethod + def from_tensors( + cls, + predictions: List[np.ndarray], + targets: List[np.ndarray], + class_idx: int, + iou_threshold: float = 0.5, + ) -> AveragePrecision: + + cls._validate_input_tensors(predictions, targets) + + evaluated_detections = np.zeros((0, 3)) + + for true_batch, detection_batch in zip(targets, predictions): + + batch_evaluated_detection = AveragePrecision.evaluate_detection_batch( + targets=true_batch, + predictions=detection_batch, + class_idx=class_idx, + iou_threshold=iou_threshold + ) + + evaluated_detections = np.concatenate([evaluated_detections, batch_evaluated_detection]) + + evaluated_detections = evaluated_detections[evaluated_detections[:, 0].argsort()[::-1]] + tp = np.cumsum(evaluated_detections[:, 1]) + all_detections = evaluated_detections.shape[0] + precision = tp / np.arange(1, all_detections + 1) + recall = tp / all_detections + + return cls.from_precision_recall(precision=precision, recall=recall, class_idx=class_idx, iou_threshold=iou_threshold) + + @classmethod + def from_precision_recall( + cls, + recall: np.ndarray, + precision: np.ndarray, + class_idx: Optional[int] = None, + iou_threshold: Optional[float] = None + ) -> AveragePrecision: + """ + Calculate average precision (AP) metric based on given precision/recall curve. + """ + EPSILON = 1e-6 + if precision.shape[0] == 0: + recall_values = np.array([0., EPSILON]) + precision_values = np.array([1., 0.]) + else: + recall_values = np.concatenate(([0.], recall, [1.0])) # as per yolov8 max= + precision_values = np.concatenate(([1.], precision, [0.])) + + precision_values = np.flip(np.maximum.accumulate(np.flip(precision_values))) + i = np.where(recall_values[1:] != recall_values[:-1])[0] + value = np.sum((recall_values[i + 1] - recall_values[i]) * precision_values[i + 1]) + return cls( + value=value, + recall_values=recall_values, + precision_values=precision_values, + class_idx=class_idx, + iou_threshold=iou_threshold + ) + + @classmethod + def _validate_input_tensors( + cls, predictions: List[np.ndarray], targets: List[np.ndarray] + ): + """ + Checks for shape consistency of input tensors. + """ + if len(predictions) != len(targets): + raise ValueError( + f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." + ) + if len(predictions) > 0: + if not isinstance(predictions[0], np.ndarray) or not isinstance( + targets[0], np.ndarray + ): + raise ValueError( + f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." + ) + if predictions[0].shape[1] != 6: + raise ValueError( + f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." + ) + if targets[0].shape[1] != 5: + raise ValueError( + f"Targets must have shape (N, 5). Got {targets[0].shape} instead." + ) + + @staticmethod + def evaluate_detection_batch( + predictions: np.ndarray, + targets: np.ndarray, + class_idx: int, + iou_threshold: float, + ) -> np.ndarray: + + detection_batch_filtered = predictions[predictions[:, 4] == class_idx] + true_batch_filtered = targets[targets[:, 4] == class_idx] + # confidence, tp, fp + result_matrix = np.zeros((detection_batch_filtered.shape[0], 3)) + + true_boxes = true_batch_filtered[:, :4] + detection_boxes = detection_batch_filtered[:, :4] + detection_conf = detection_batch_filtered[:, 5] + iou_batch = box_iou_batch(boxes_true=true_boxes, boxes_detection=detection_boxes) + matched_idx = np.asarray(iou_batch > iou_threshold).nonzero() + + if matched_idx[0].shape[0]: + matches = np.stack((matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1) + matches = AveragePrecision._drop_extra_matches(matches=matches) + else: + matches = np.zeros((0, 3)) + + matched_true_class_idx, matched_detection_class_idx, _ = matches.transpose().astype(np.int16) + + for i, conf in enumerate(detection_conf): + if any(matched_detection_class_idx == i): + result_matrix[i] = np.array([conf, 1, 0]) + else: + result_matrix[i] = np.array([conf, 0, 1]) + + return result_matrix + + @staticmethod + def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: + """ + Deduplicate matches. If there are multiple matches for the same true or predicted box, + only the one with the highest IoU is kept. + """ + if matches.shape[0] > 0: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + return matches + + +@dataclass(frozen=True) +class MeanAveragePrecision: + value: float + per_class: List[AveragePrecision] + class_names: List[str] + iou_threshold: float + + @classmethod + def from_detections( + cls, + predictions: List[Detections], + targets: List[Detections], + class_names: List[str], + iou_threshold: float = 0.6 + ) -> MeanAveragePrecision: + num_classes = len(class_names) + per_class = [ + AveragePrecision.from_detections( + targets=targets, + predictions=predictions, + class_idx=class_idx, + iou_threshold=iou_threshold + ) + for class_idx + in range(num_classes) + ] + values = [ap.value for ap in per_class] + + print(values) + return cls(value=sum(values) / num_classes, per_class=per_class, class_names=class_names, iou_threshold=iou_threshold) + + def plot(self, target_path: str, title: Optional[str] = None, class_names: Optional[List[str]] = None) -> None: + """ + Create mean average precision plot and save it at selected location. + + Args: + target_path: `str` selected target location of confusion matrix plot. + title: `Optional[str]` title displayed at the top of the confusion matrix plot. Default `None`. + class_names: `Optional[List[str]]` list of class names detected my model. If non given class indexes will be used. Default `None`. + """ + text_labels = class_names is not None and len(class_names) == self.num_classes + labels = class_names if text_labels else list(range(self.num_classes)) + + fig = plt.figure(figsize=(12, 9), tight_layout=True, facecolor='white') + ax = fig.add_subplot(111) + + for label, ap in zip(labels, self.per_class): + ax.plot(ap.recall_values, ap.precision_values, label=label, linewidth=2.0,) + + plt.xlabel('Recall') + plt.ylabel('Precision') + plt.xlim([0, 1]) + plt.ylim([0, 1]) + + # axis style + for spine in ax.spines.values(): + spine.set_edgecolor('black') + for s in ['top', 'right']: + ax.spines[s].set_visible(False) + ax.spines[s].set_visible(False) + + handles, labels = ax.get_legend_handles_labels() + ax.legend(handles, labels, loc='upper center', bbox_to_anchor=(1.1, 1), facecolor='white', + framealpha=1, frameon=False, fontsize=10) + ax.set_facecolor('white') + ax.grid(b=True, color='grey', linestyle='-.', linewidth=0.5, alpha=0.5) + + if title: + plt.title(title, fontsize=20, pad=20) + + fig.savefig(target_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True) From 0f95c5d8b38f7cf49cece071bc954f2577e67c74 Mon Sep 17 00:00:00 2001 From: hd Date: Sun, 23 Jul 2023 16:44:14 +0200 Subject: [PATCH 05/54] update --- supervision/__init__.py | 2 +- supervision/metrics/detection.py | 348 ++++++++++++++++--------------- 2 files changed, 178 insertions(+), 172 deletions(-) diff --git a/supervision/__init__.py b/supervision/__init__.py index b817d113..edb9675d 100644 --- a/supervision/__init__.py +++ b/supervision/__init__.py @@ -30,7 +30,7 @@ from supervision.draw.color import Color, ColorPalette from supervision.draw.utils import draw_filled_rectangle, draw_polygon, draw_text from supervision.geometry.core import Point, Position, Rect from supervision.geometry.utils import get_polygon_center -from supervision.metrics.detection import ConfusionMatrix, AveragePrecision, MeanAveragePrecision +from supervision.metrics.detection import ConfusionMatrix, MeanAveragePrecision from supervision.utils.file import list_files_with_extensions from supervision.utils.image import ImageSink, crop from supervision.utils.notebook import plot_image, plot_images_grid diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 10704525..f794c1e0 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -10,7 +10,7 @@ import numpy as np from supervision.dataset.core import DetectionDataset from supervision.detection.core import Detections from supervision.detection.utils import box_iou_batch - +import torch @dataclass class ConfusionMatrix: @@ -102,6 +102,13 @@ class ConfusionMatrix: def detections_to_tensor( cls, detections: Detections, with_confidence: bool = False ) -> np.ndarray: + + if detections == Detections.empty(): + if with_confidence: + return np.zeros((0, 6)) + else: + return np.zeros((0, 5)) + if detections.class_id is None: raise ValueError( "ConfusionMatrix can only be calculated for Detections with class_id" @@ -458,125 +465,143 @@ class ConfusionMatrix: return fig -@dataclass -class AveragePrecision: - value: float - recall_values: np.ndarray - precision_values: np.ndarray - class_idx: Optional[int] = None - iou_threshold: Optional[float] = None + +@dataclass(frozen=True) +class MeanAveragePrecision: + map: float + map50: float + ap50: float + per_class: List[np.ndarray] + classes: List[str] @classmethod def from_detections( cls, predictions: List[Detections], targets: List[Detections], - class_idx: int, - iou_threshold: float = 0.5, - ) -> AveragePrecision: + classes: List[str], + ): prediction_tensors = [] target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - cls.detections_to_tensor(prediction, with_confidence=True) + MeanAveragePrecision.detections_to_tensor(prediction, with_confidence=True) ) target_tensors.append( - cls.detections_to_tensor(target, with_confidence=False) + MeanAveragePrecision.detections_to_tensor(target, with_confidence=False) ) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, - class_idx=class_idx, - iou_threshold=iou_threshold, + classes=classes, ) - @classmethod + + @staticmethod def detections_to_tensor( - cls, detections: Detections, with_confidence: bool = False + detections: Detections, with_confidence: bool = False ) -> np.ndarray: + if detections == Detections.empty(): + return np.zeros((0, 6)) if with_confidence else np.zeros((0, 5)) + if detections.class_id is None: raise ValueError( - "ConfusionMatrix can only be calculated for Detections with class_id" + "MeanAveragePrecision can only be calculated for Detections with class_id" ) arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] if with_confidence: if detections.confidence is None: - raise ValueError( - "ConfusionMatrix can only be calculated for Detections with confidence" - ) - arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + arrays_to_concat.append(np.zeros((0, 2))) + else: + arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) return np.concatenate(arrays_to_concat, axis=1) @classmethod def from_tensors( - cls, - predictions: List[np.ndarray], - targets: List[np.ndarray], - class_idx: int, - iou_threshold: float = 0.5, - ) -> AveragePrecision: - + cls, + predictions: List[np.ndarray], + targets: List[np.ndarray], + classes: List[str], + ): cls._validate_input_tensors(predictions, targets) - evaluated_detections = np.zeros((0, 3)) + tp, fp, p, r, f1, mp, mr, map50, ap50, map = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + stats, ap, ap_class = [], [], [] + num_classes = len(classes) + correct_boxes = [] + iouv = np.linspace(0.5, 0.95, 10) for true_batch, detection_batch in zip(targets, predictions): + nl, npr = detection_batch.shape[0], true_batch.shape[0] # number of labels, predictions + correct = np.zeros((detection_batch.shape[0], iouv.shape[0])).astype(bool) + if npr == 0: + if nl: + stats.append((torch.from_numpy(correct), *torch.zeros((2, 0), device='cpu'), torch.from_numpy(true_batch[:, -1]))) + # stats.append([correct, np.zeros((0, 2)), np.zeros((0, 2)), np.expand_dims(true_batch[:, -1], 1)]) + continue - batch_evaluated_detection = AveragePrecision.evaluate_detection_batch( - targets=true_batch, + correct_boxes.append(cls.evaluate_detection_batch( predictions=detection_batch, - class_idx=class_idx, - iou_threshold=iou_threshold - ) + targets=true_batch, + )) + stats.append((torch.from_numpy(correct), torch.from_numpy(detection_batch[:, 4]), torch.from_numpy(detection_batch[:, -1]), torch.from_numpy(true_batch[:, -1]))) + # stats.append([correct, np.expand_dims(detection_batch[:, 4], 1), np.expand_dims(detection_batch[:, -1], 1), np.expand_dims(true_batch[:, -1], 1)]) - evaluated_detections = np.concatenate([evaluated_detections, batch_evaluated_detection]) - evaluated_detections = evaluated_detections[evaluated_detections[:, 0].argsort()[::-1]] - tp = np.cumsum(evaluated_detections[:, 1]) - all_detections = evaluated_detections.shape[0] - precision = tp / np.arange(1, all_detections + 1) - recall = tp / all_detections + stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy - return cls.from_precision_recall(precision=precision, recall=recall, class_idx=class_idx, iou_threshold=iou_threshold) + if len(stats) and len(stats[0]): + tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, names=classes) + print(ap_class) + ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 + mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() - @classmethod - def from_precision_recall( - cls, - recall: np.ndarray, - precision: np.ndarray, - class_idx: Optional[int] = None, - iou_threshold: Optional[float] = None - ) -> AveragePrecision: - """ - Calculate average precision (AP) metric based on given precision/recall curve. - """ - EPSILON = 1e-6 - if precision.shape[0] == 0: - recall_values = np.array([0., EPSILON]) - precision_values = np.array([1., 0.]) - else: - recall_values = np.concatenate(([0.], recall, [1.0])) # as per yolov8 max= - precision_values = np.concatenate(([1.], precision, [0.])) + return cls(map=map, map50=map50, ap50=ap50, per_class=ap_class, classes=classes) + # return cls( + # matrix=matrix, + # classes=classes, + # conf_threshold=conf_threshold, + # iou_threshold=iou_threshold, + # ) - precision_values = np.flip(np.maximum.accumulate(np.flip(precision_values))) - i = np.where(recall_values[1:] != recall_values[:-1])[0] - value = np.sum((recall_values[i + 1] - recall_values[i]) * precision_values[i + 1]) - return cls( - value=value, - recall_values=recall_values, - precision_values=precision_values, - class_idx=class_idx, - iou_threshold=iou_threshold - ) + @staticmethod + def evaluate_detection_batch( + predictions: np.ndarray, + targets: np.ndarray, + ) -> np.ndarray: + + iouv = np.linspace(0.5, 0.95, 10) + + iou_batch = box_iou_batch(targets[:, :4], predictions[:, :4]) + correct = np.zeros((predictions.shape[0], iouv.shape[0])).astype(bool) + # correct_class = targets[:, -1] == predictions[:, -1] + + for i in range(len(iouv)): + # todo add correct class here + # x = np.where(iou >= iouv[i] and correct_class) + matched_idx = np.asarray(iou_batch > iouv[i]).nonzero() + + if matched_idx[0].shape[0]: + # matches = np.hstack() + # print(iou[x[0], x[1]][:, None].shape) # (n, 1) + matches = np.stack((matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1) + + if matched_idx[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + # matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + return correct @classmethod def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] + cls, predictions: List[np.ndarray], targets: List[np.ndarray] ): """ Checks for shape consistency of input tensors. @@ -587,7 +612,7 @@ class AveragePrecision: ) if len(predictions) > 0: if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray + targets[0], np.ndarray ): raise ValueError( f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." @@ -601,47 +626,8 @@ class AveragePrecision: f"Targets must have shape (N, 5). Got {targets[0].shape} instead." ) - @staticmethod - def evaluate_detection_batch( - predictions: np.ndarray, - targets: np.ndarray, - class_idx: int, - iou_threshold: float, - ) -> np.ndarray: - - detection_batch_filtered = predictions[predictions[:, 4] == class_idx] - true_batch_filtered = targets[targets[:, 4] == class_idx] - # confidence, tp, fp - result_matrix = np.zeros((detection_batch_filtered.shape[0], 3)) - - true_boxes = true_batch_filtered[:, :4] - detection_boxes = detection_batch_filtered[:, :4] - detection_conf = detection_batch_filtered[:, 5] - iou_batch = box_iou_batch(boxes_true=true_boxes, boxes_detection=detection_boxes) - matched_idx = np.asarray(iou_batch > iou_threshold).nonzero() - - if matched_idx[0].shape[0]: - matches = np.stack((matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1) - matches = AveragePrecision._drop_extra_matches(matches=matches) - else: - matches = np.zeros((0, 3)) - - matched_true_class_idx, matched_detection_class_idx, _ = matches.transpose().astype(np.int16) - - for i, conf in enumerate(detection_conf): - if any(matched_detection_class_idx == i): - result_matrix[i] = np.array([conf, 1, 0]) - else: - result_matrix[i] = np.array([conf, 0, 1]) - - return result_matrix - @staticmethod def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: - """ - Deduplicate matches. If there are multiple matches for the same true or predicted box, - only the one with the highest IoU is kept. - """ if matches.shape[0] > 0: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] @@ -650,74 +636,94 @@ class AveragePrecision: return matches -@dataclass(frozen=True) -class MeanAveragePrecision: - value: float - per_class: List[AveragePrecision] - class_names: List[str] - iou_threshold: float +def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + # Arguments + tp: True positives (nparray, nx1 or nx10). + conf: Objectness value from 0-1 (nparray). + pred_cls: Predicted object classes (nparray). + target_cls: True object classes (nparray). + plot: Plot precision-recall curve at mAP@0.5 + save_dir: Plot save directory + # Returns + The average precision as computed in py-faster-rcnn. + """ - @classmethod - def from_detections( - cls, - predictions: List[Detections], - targets: List[Detections], - class_names: List[str], - iou_threshold: float = 0.6 - ) -> MeanAveragePrecision: - num_classes = len(class_names) - per_class = [ - AveragePrecision.from_detections( - targets=targets, - predictions=predictions, - class_idx=class_idx, - iou_threshold=iou_threshold - ) - for class_idx - in range(num_classes) - ] - values = [ap.value for ap in per_class] + # Sort by objectness + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] - print(values) - return cls(value=sum(values) / num_classes, per_class=per_class, class_names=class_names, iou_threshold=iou_threshold) + # Find unique classes + unique_classes, nt = np.unique(target_cls, return_counts=True) + nc = unique_classes.shape[0] # number of classes, number of detections - def plot(self, target_path: str, title: Optional[str] = None, class_names: Optional[List[str]] = None) -> None: - """ - Create mean average precision plot and save it at selected location. + # Create Precision-Recall curve and compute AP for each class + px, py = np.linspace(0, 1, 1000), [] # for plotting + ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_l = nt[ci] # number of labels + n_p = i.sum() # number of predictions + if n_p == 0 or n_l == 0: + continue - Args: - target_path: `str` selected target location of confusion matrix plot. - title: `Optional[str]` title displayed at the top of the confusion matrix plot. Default `None`. - class_names: `Optional[List[str]]` list of class names detected my model. If non given class indexes will be used. Default `None`. - """ - text_labels = class_names is not None and len(class_names) == self.num_classes - labels = class_names if text_labels else list(range(self.num_classes)) + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) - fig = plt.figure(figsize=(12, 9), tight_layout=True, facecolor='white') - ax = fig.add_subplot(111) + # Recall + recall = tpc / (n_l + eps) # recall curve + r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases - for label, ap in zip(labels, self.per_class): - ax.plot(ap.recall_values, ap.precision_values, label=label, linewidth=2.0,) + # Precision + precision = tpc / (tpc + fpc) # precision curve + p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score - plt.xlabel('Recall') - plt.ylabel('Precision') - plt.xlim([0, 1]) - plt.ylim([0, 1]) + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) - # axis style - for spine in ax.spines.values(): - spine.set_edgecolor('black') - for s in ['top', 'right']: - ax.spines[s].set_visible(False) - ax.spines[s].set_visible(False) + # Compute F1 (harmonic mean of precision and recall) + f1 = 2 * p * r / (p + r + eps) - handles, labels = ax.get_legend_handles_labels() - ax.legend(handles, labels, loc='upper center', bbox_to_anchor=(1.1, 1), facecolor='white', - framealpha=1, frameon=False, fontsize=10) - ax.set_facecolor('white') - ax.grid(b=True, color='grey', linestyle='-.', linewidth=0.5, alpha=0.5) + i = smooth(f1.mean(0), 0.1).argmax() # max F1 index + p, r, f1 = p[:, i], r[:, i], f1[:, i] + tp = (r * nt).round() # true positives + fp = (tp / (p + eps) - tp).round() # false positives + return tp, fp, p, r, f1, ap, unique_classes.astype(int) - if title: - plt.title(title, fontsize=20, pad=20) +def smooth(y, f=0.05): + # Box filter of fraction f + nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) + p = np.ones(nf // 2) # ones padding + yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded + return np.convolve(yp, np.ones(nf) / nf, mode='valid') # y-smoothed + +def compute_ap(recall, precision): + """ Compute the average precision, given the recall and precision curves + # Arguments + recall: The recall curve (list) + precision: The precision curve (list) + # Returns + Average precision, precision curve, recall curve + """ + + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.0], recall, [1.0])) + mpre = np.concatenate(([1.0], precision, [0.0])) + + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + + # Integrate area under curve + method = 'interp' # methods: 'continuous', 'interp' + if method == 'interp': + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve + + return ap, mpre, mrec - fig.savefig(target_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True) From 0cfad8407c4194354fcd765d4491401a5d022143 Mon Sep 17 00:00:00 2001 From: hd Date: Sun, 23 Jul 2023 22:27:47 +0200 Subject: [PATCH 06/54] update --- supervision/dataset/core.py | 20 ----- supervision/metrics/detection.py | 149 ++++++++++++++++--------------- 2 files changed, 75 insertions(+), 94 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index a081d2fb..ce7e0edb 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -46,14 +46,6 @@ class BaseDataset(ABC): ) -> Tuple[BaseDataset, BaseDataset]: pass - def add_instance( - self, filename: str, image: np.ndarray, detections: Detections - ) -> None: - pass - - def add_class_names(self, class_names: List[str]) -> None: - pass - @dataclass class DetectionDataset(BaseDataset): @@ -511,18 +503,6 @@ class DetectionDataset(BaseDataset): classes=merged_classes, images=merged_images, annotations=merged_annotations ) - def add_class_names(self, class_names: List[str]): - self.classes = class_names - - def add_instance(self, filename: str, image: np.ndarray, detections: Detections): - if filename not in self.annotations.keys(): - self.images[filename] = image - self.annotations[filename] = detections - - @classmethod - def emty(cls): - return cls(classes=[], images={}, annotations={}) - @dataclass class ClassificationDataset(BaseDataset): diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index f794c1e0..7171bcf6 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -10,7 +10,7 @@ import numpy as np from supervision.dataset.core import DetectionDataset from supervision.detection.core import Detections from supervision.detection.utils import box_iou_batch -import torch + @dataclass class ConfusionMatrix: @@ -102,7 +102,6 @@ class ConfusionMatrix: def detections_to_tensor( cls, detections: Detections, with_confidence: bool = False ) -> np.ndarray: - if detections == Detections.empty(): if with_confidence: return np.zeros((0, 6)) @@ -464,9 +463,7 @@ class ConfusionMatrix: ) return fig - - - +import torch @dataclass(frozen=True) class MeanAveragePrecision: map: float @@ -481,13 +478,14 @@ class MeanAveragePrecision: predictions: List[Detections], targets: List[Detections], classes: List[str], - ): - + ) -> MeanAveragePrecision: prediction_tensors = [] target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - MeanAveragePrecision.detections_to_tensor(prediction, with_confidence=True) + MeanAveragePrecision.detections_to_tensor( + prediction, with_confidence=True + ) ) target_tensors.append( MeanAveragePrecision.detections_to_tensor(target, with_confidence=False) @@ -498,14 +496,10 @@ class MeanAveragePrecision: classes=classes, ) - @staticmethod def detections_to_tensor( - detections: Detections, with_confidence: bool = False + detections: Detections, with_confidence: bool = False ) -> np.ndarray: - if detections == Detections.empty(): - return np.zeros((0, 6)) if with_confidence else np.zeros((0, 5)) - if detections.class_id is None: raise ValueError( "MeanAveragePrecision can only be calculated for Detections with class_id" @@ -514,94 +508,98 @@ class MeanAveragePrecision: arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] if with_confidence: - if detections.confidence is None: - arrays_to_concat.append(np.zeros((0, 2))) - else: + if detections.confidence is not None: arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) return np.concatenate(arrays_to_concat, axis=1) @classmethod def from_tensors( - cls, - predictions: List[np.ndarray], - targets: List[np.ndarray], - classes: List[str], - ): + cls, + predictions: List[np.ndarray], + targets: List[np.ndarray], + classes: List[str], + ) -> MeanAveragePrecision: cls._validate_input_tensors(predictions, targets) - tp, fp, p, r, f1, mp, mr, map50, ap50, map = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 - stats, ap, ap_class = [], [], [] - num_classes = len(classes) - correct_boxes = [] + tp, fp, p, r, f1, mp, mr, map50, ap50, map = ( + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ) + jdict, stats, ap, ap_class = [], [], [], [] + iouv = np.linspace(0.5, 0.95, 10) + niou = iouv.size for true_batch, detection_batch in zip(targets, predictions): - nl, npr = detection_batch.shape[0], true_batch.shape[0] # number of labels, predictions - correct = np.zeros((detection_batch.shape[0], iouv.shape[0])).astype(bool) + nl, npr = ( + true_batch.shape[0], + detection_batch.shape[0], + ) # number of labels, predictions + correct = np.zeros((npr, niou), dtype=bool) # init + if npr == 0: if nl: - stats.append((torch.from_numpy(correct), *torch.zeros((2, 0), device='cpu'), torch.from_numpy(true_batch[:, -1]))) - # stats.append([correct, np.zeros((0, 2)), np.zeros((0, 2)), np.expand_dims(true_batch[:, -1], 1)]) + stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) continue + if nl: + correct, iouv = cls.evaluate_detection_batch( + predictions=detection_batch, + targets=true_batch, + iouv=iouv, + ) + # (correct, conf, pcls, tcls) + stats.append( + ( + correct, + detection_batch[:, 5], + detection_batch[:, 4], + true_batch[:, 4], + ) + ) - correct_boxes.append(cls.evaluate_detection_batch( - predictions=detection_batch, - targets=true_batch, - )) - stats.append((torch.from_numpy(correct), torch.from_numpy(detection_batch[:, 4]), torch.from_numpy(detection_batch[:, -1]), torch.from_numpy(true_batch[:, -1]))) - # stats.append([correct, np.expand_dims(detection_batch[:, 4], 1), np.expand_dims(detection_batch[:, -1], 1), np.expand_dims(true_batch[:, -1], 1)]) + stats = [np.concatenate(x, 0) for x in zip(*stats)] - - stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy - - if len(stats) and len(stats[0]): + if len(stats) and stats[0].any(): tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, names=classes) - print(ap_class) ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() return cls(map=map, map50=map50, ap50=ap50, per_class=ap_class, classes=classes) - # return cls( - # matrix=matrix, - # classes=classes, - # conf_threshold=conf_threshold, - # iou_threshold=iou_threshold, - # ) @staticmethod def evaluate_detection_batch( - predictions: np.ndarray, - targets: np.ndarray, + predictions: np.ndarray, targets: np.ndarray, iouv: np.ndarray ) -> np.ndarray: - - iouv = np.linspace(0.5, 0.95, 10) - - iou_batch = box_iou_batch(targets[:, :4], predictions[:, :4]) correct = np.zeros((predictions.shape[0], iouv.shape[0])).astype(bool) - # correct_class = targets[:, -1] == predictions[:, -1] + iou = box_iou_batch(targets[:, :4], predictions[:, :4]) + correct_class = targets[:, 4:5] == predictions[:, 4] for i in range(len(iouv)): - # todo add correct class here - # x = np.where(iou >= iouv[i] and correct_class) - matched_idx = np.asarray(iou_batch > iouv[i]).nonzero() + x = np.where((iou >= iouv[i]) & correct_class) - if matched_idx[0].shape[0]: - # matches = np.hstack() - # print(iou[x[0], x[1]][:, None].shape) # (n, 1) - matches = np.stack((matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1) - - if matched_idx[0].shape[0] > 1: + if x[0].shape[0]: + _X1 = np.vstack([x[0], x[1]]).T + _x2 = iou[x[0], x[1]][:, None] + matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] + if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - # matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True - return correct + return correct, iouv @classmethod def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] + cls, predictions: List[np.ndarray], targets: List[np.ndarray] ): """ Checks for shape consistency of input tensors. @@ -612,7 +610,7 @@ class MeanAveragePrecision: ) if len(predictions) > 0: if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray + targets[0], np.ndarray ): raise ValueError( f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." @@ -637,7 +635,7 @@ class MeanAveragePrecision: def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): - """ Compute the average precision, given the recall and precision curves. + """Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. # Arguments tp: True positives (nparray, nx1 or nx10). @@ -674,7 +672,9 @@ def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): # Recall recall = tpc / (n_l + eps) # recall curve - r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases + r[ci] = np.interp( + -px, -conf[i], recall[:, 0], left=0 + ) # negative x, xp because xp decreases # Precision precision = tpc / (tpc + fpc) # precision curve @@ -691,17 +691,19 @@ def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): p, r, f1 = p[:, i], r[:, i], f1[:, i] tp = (r * nt).round() # true positives fp = (tp / (p + eps) - tp).round() # false positives - return tp, fp, p, r, f1, ap, unique_classes.astype(int) + return ap, unique_classes.astype(int) + def smooth(y, f=0.05): # Box filter of fraction f nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) p = np.ones(nf // 2) # ones padding yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded - return np.convolve(yp, np.ones(nf) / nf, mode='valid') # y-smoothed + return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed + def compute_ap(recall, precision): - """ Compute the average precision, given the recall and precision curves + """Compute the average precision, given the recall and precision curves # Arguments recall: The recall curve (list) precision: The precision curve (list) @@ -717,8 +719,8 @@ def compute_ap(recall, precision): mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) # Integrate area under curve - method = 'interp' # methods: 'continuous', 'interp' - if method == 'interp': + method = "interp" # methods: 'continuous', 'interp' + if method == "interp": x = np.linspace(0, 1, 101) # 101-point interp (COCO) ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate else: # 'continuous' @@ -726,4 +728,3 @@ def compute_ap(recall, precision): ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve return ap, mpre, mrec - From 40a6fe1b5ff2c256695d282e4ef2b86d94bec87e Mon Sep 17 00:00:00 2001 From: hd Date: Sun, 23 Jul 2023 22:35:51 +0200 Subject: [PATCH 07/54] update --- supervision/metrics/detection.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 7171bcf6..de1ffe47 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -463,7 +463,10 @@ class ConfusionMatrix: ) return fig + import torch + + @dataclass(frozen=True) class MeanAveragePrecision: map: float @@ -556,7 +559,7 @@ class MeanAveragePrecision: targets=true_batch, iouv=iouv, ) - # (correct, conf, pcls, tcls) + # (correct, confidence, pred-class, target-class) stats.append( ( correct, @@ -590,10 +593,7 @@ class MeanAveragePrecision: _X1 = np.vstack([x[0], x[1]]).T _x2 = iou[x[0], x[1]][:, None] matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] - if x[0].shape[0] > 1: - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + matches = MeanAveragePrecision._drop_extra_matches(matches) correct[matches[:, 1].astype(int), i] = True return correct, iouv @@ -634,29 +634,13 @@ class MeanAveragePrecision: return matches -def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): - """Compute the average precision, given the recall and precision curves. - Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. - # Arguments - tp: True positives (nparray, nx1 or nx10). - conf: Objectness value from 0-1 (nparray). - pred_cls: Predicted object classes (nparray). - target_cls: True object classes (nparray). - plot: Plot precision-recall curve at mAP@0.5 - save_dir: Plot save directory - # Returns - The average precision as computed in py-faster-rcnn. - """ - - # Sort by objectness +def ap_per_class(tp, conf, pred_cls, target_cls, eps=1e-16): i = np.argsort(-conf) tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] - # Find unique classes unique_classes, nt = np.unique(target_cls, return_counts=True) nc = unique_classes.shape[0] # number of classes, number of detections - # Create Precision-Recall curve and compute AP for each class px, py = np.linspace(0, 1, 1000), [] # for plotting ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) for ci, c in enumerate(unique_classes): @@ -691,7 +675,7 @@ def ap_per_class(tp, conf, pred_cls, target_cls, names=(), eps=1e-16): p, r, f1 = p[:, i], r[:, i], f1[:, i] tp = (r * nt).round() # true positives fp = (tp / (p + eps) - tp).round() # false positives - return ap, unique_classes.astype(int) + return tp, fp, p, r, f1, ap, unique_classes.astype(int) def smooth(y, f=0.05): From 2f726ca795dcc8323fc0bb16cb604fcd652bc607 Mon Sep 17 00:00:00 2001 From: hd Date: Sun, 23 Jul 2023 22:37:44 +0200 Subject: [PATCH 08/54] Initial working version --- supervision/metrics/detection.py | 141 +++++++++++++++---------------- 1 file changed, 70 insertions(+), 71 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index de1ffe47..39bead32 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -464,9 +464,6 @@ class ConfusionMatrix: return fig -import torch - - @dataclass(frozen=True) class MeanAveragePrecision: map: float @@ -572,7 +569,7 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, names=classes) + tp, fp, p, r, f1, ap, ap_class = MeanAveragePrecision.ap_per_class(*stats, names=classes) ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() @@ -633,82 +630,84 @@ class MeanAveragePrecision: matches = matches[np.unique(matches[:, 0], return_index=True)[1]] return matches + @staticmethod + def compute_ap(recall, precision): + """Compute the average precision, given the recall and precision curves + # Arguments + recall: The recall curve (list) + precision: The precision curve (list) + # Returns + Average precision, precision curve, recall curve + """ + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.0], recall, [1.0])) + mpre = np.concatenate(([1.0], precision, [0.0])) -def ap_per_class(tp, conf, pred_cls, target_cls, eps=1e-16): - i = np.argsort(-conf) - tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) - unique_classes, nt = np.unique(target_cls, return_counts=True) - nc = unique_classes.shape[0] # number of classes, number of detections + # Integrate area under curve + method = "interp" # methods: 'continuous', 'interp' + if method == "interp": + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve - px, py = np.linspace(0, 1, 1000), [] # for plotting - ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) - for ci, c in enumerate(unique_classes): - i = pred_cls == c - n_l = nt[ci] # number of labels - n_p = i.sum() # number of predictions - if n_p == 0 or n_l == 0: - continue + return ap, mpre, mrec - # Accumulate FPs and TPs - fpc = (1 - tp[i]).cumsum(0) - tpc = tp[i].cumsum(0) + @staticmethod + def ap_per_class(tp, conf, pred_cls, target_cls, eps=1e-16): + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] - # Recall - recall = tpc / (n_l + eps) # recall curve - r[ci] = np.interp( - -px, -conf[i], recall[:, 0], left=0 - ) # negative x, xp because xp decreases + unique_classes, nt = np.unique(target_cls, return_counts=True) + nc = unique_classes.shape[0] # number of classes, number of detections - # Precision - precision = tpc / (tpc + fpc) # precision curve - p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score + px, py = np.linspace(0, 1, 1000), [] # for plotting + ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_l = nt[ci] # number of labels + n_p = i.sum() # number of predictions + if n_p == 0 or n_l == 0: + continue - # AP from recall-precision curve - for j in range(tp.shape[1]): - ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) - # Compute F1 (harmonic mean of precision and recall) - f1 = 2 * p * r / (p + r + eps) + # Recall + recall = tpc / (n_l + eps) # recall curve + r[ci] = np.interp( + -px, -conf[i], recall[:, 0], left=0 + ) # negative x, xp because xp decreases - i = smooth(f1.mean(0), 0.1).argmax() # max F1 index - p, r, f1 = p[:, i], r[:, i], f1[:, i] - tp = (r * nt).round() # true positives - fp = (tp / (p + eps) - tp).round() # false positives - return tp, fp, p, r, f1, ap, unique_classes.astype(int) + # Precision + precision = tpc / (tpc + fpc) # precision curve + p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score + + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j], mpre, mrec = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) + + # Compute F1 (harmonic mean of precision and recall) + f1 = 2 * p * r / (p + r + eps) + + i = MeanAveragePrecision.smooth(f1.mean(0), 0.1).argmax() # max F1 index + p, r, f1 = p[:, i], r[:, i], f1[:, i] + tp = (r * nt).round() # true positives + fp = (tp / (p + eps) - tp).round() # false positives + return tp, fp, p, r, f1, ap, unique_classes.astype(int) + + @staticmethod + def smooth(y, f=0.05): + # Box filter of fraction f + nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) + p = np.ones(nf // 2) # ones padding + yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded + return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed -def smooth(y, f=0.05): - # Box filter of fraction f - nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) - p = np.ones(nf // 2) # ones padding - yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded - return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed - -def compute_ap(recall, precision): - """Compute the average precision, given the recall and precision curves - # Arguments - recall: The recall curve (list) - precision: The precision curve (list) - # Returns - Average precision, precision curve, recall curve - """ - - # Append sentinel values to beginning and end - mrec = np.concatenate(([0.0], recall, [1.0])) - mpre = np.concatenate(([1.0], precision, [0.0])) - - # Compute the precision envelope - mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) - - # Integrate area under curve - method = "interp" # methods: 'continuous', 'interp' - if method == "interp": - x = np.linspace(0, 1, 101) # 101-point interp (COCO) - ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate - else: # 'continuous' - i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes - ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve - - return ap, mpre, mrec From e249ea864429d77da6ec70614076301e300d71d4 Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 16:11:16 +0200 Subject: [PATCH 09/54] update --- supervision/metrics/detection.py | 292 +++++++++++++++++++++++-------- 1 file changed, 215 insertions(+), 77 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 39bead32..4c34c5a4 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -468,9 +468,19 @@ class ConfusionMatrix: class MeanAveragePrecision: map: float map50: float - ap50: float + map75: float per_class: List[np.ndarray] classes: List[str] + """ + Mean Average Precision for object detection tasks. + + Attributes: + map (float): map value. + map50 (float): map value at iou threshold=0.5. + map75 (float): map value at iou threshold=0.75. + per_class (np.ndarray): numpy array of map values for each class + classes (List[str]): Model class names. + """ @classmethod def from_detections( @@ -479,6 +489,43 @@ class MeanAveragePrecision: targets: List[Detections], classes: List[str], ) -> MeanAveragePrecision: + """ + Calculate confusion matrix based on predicted and ground-truth detections. + + Args: + targets (List[Detections]): Detections objects from ground-truth. + predictions (List[Detections]): Detections objects predicted by the model. + classes (List[str]): Model class names. + conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. + iou_threshold (float): Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. + + Returns: + ConfusionMatrix: New instance of ConfusionMatrix. + + Example: + ```python + >>> import supervision as sv + + >>> targets = [ + ... sv.Detections(...), + ... sv.Detections(...) + ... ] + + >>> predictions = [ + ... sv.Detections(...), + ... sv.Detections(...) + ... ] + + >>> mean_average_precison = sv.MeanAveragePrecision.from_detections( + ... predictions=predictions, + ... targets=target, + ... classes=['person', ...] + ... ) + + >>> mean_average_precison.matrix + 0.433 + ``` + """ prediction_tensors = [] target_tensors = [] for prediction, target in zip(predictions, targets): @@ -496,22 +543,53 @@ class MeanAveragePrecision: classes=classes, ) - @staticmethod - def detections_to_tensor( - detections: Detections, with_confidence: bool = False - ) -> np.ndarray: - if detections.class_id is None: - raise ValueError( - "MeanAveragePrecision can only be calculated for Detections with class_id" - ) + @classmethod + def benchmark( + cls, + dataset: DetectionDataset, + callback: Callable[[np.ndarray], Detections], + ) -> MeanAveragePrecision: + """ + Get map from dataset and callback function. - arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + Args: + dataset (DetectionDataset): Object detection dataset used for evaluation. + callback (Callable[[np.ndarray], Detections]): Function that takes an image as input and returns Detections object. + Returns: + MeanAveragePrecision: New instance of MeanAveragePrecision. - if with_confidence: - if detections.confidence is not None: - arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + Example: + ```python + >>> import supervision as sv + >>> from ultralytics import YOLO - return np.concatenate(arrays_to_concat, axis=1) + >>> dataset = sv.DetectionDataset.from_yolo(...) + + >>> model = YOLO(...) + >>> def callback(image: np.ndarray) -> sv.Detections: + ... result = model(image)[0] + ... return sv.Detections.from_yolov8(result) + + >>> mean_average_precision = sv.MeanAveragePrecision.benchmark( + ... dataset = dataset, + ... callback = callback + ... ) + + >>> mean_average_precision.map + 0.433 + ``` + """ + predictions, targets = [], [] + for img_name, img in dataset.images.items(): + predictions_batch = callback(img) + predictions.append(predictions_batch) + targets_batch = dataset.annotations[img_name] + targets.append(targets_batch) + return cls.from_detections( + predictions=predictions, + targets=targets, + classes=dataset.classes, + ) @classmethod def from_tensors( @@ -520,22 +598,61 @@ class MeanAveragePrecision: targets: List[np.ndarray], classes: List[str], ) -> MeanAveragePrecision: + """ + Calculate Mean Average Precision based on predicted and ground-truth detections at different threshold. + + Args: + predictions (List[np.ndarray]): Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. + targets (List[np.ndarray]): Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. + classes (List[str]): Model class names. + Returns: + MeanAveragePrecision: New instance of MeanAveragePrecision. + + Example: + ```python + >>> import supervision as sv + + >>> targets = ( + ... [ + ... array( + ... [ + ... [0.0, 0.0, 3.0, 3.0, 1], + ... [2.0, 2.0, 5.0, 5.0, 1], + ... [6.0, 1.0, 8.0, 3.0, 2], + ... ] + ... ), + ... array([1.0, 1.0, 2.0, 2.0, 2]), + ... ] + ... ) + + >>> predictions = [ + ... array( + ... [ + ... [0.0, 0.0, 3.0, 3.0, 1, 0.9], + ... [0.1, 0.1, 3.0, 3.0, 0, 0.9], + ... [6.0, 1.0, 8.0, 3.0, 1, 0.8], + ... [1.0, 6.0, 2.0, 7.0, 1, 0.8], + ... ] + ... ), + ... array([[1.0, 1.0, 2.0, 2.0, 2, 0.8]]) + ... ] + + >>> mean_average_precison = sv.MeanAveragePrecision.from_tensors( + ... predictions=predictions, + ... targets=targets, + ... classes=['person', ...] + ... ) + + >>> mean_average_precison.map + 0.433 + ``` + """ cls._validate_input_tensors(predictions, targets) + map, map50, map75, map90 = 0, 0, 0, 0 + ap50, ap75, ap90 = 0, 0, 0 + p, r, f1 = 0, 0, 0 - tp, fp, p, r, f1, mp, mr, map50, ap50, map = ( - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - ) - jdict, stats, ap, ap_class = [], [], [], [] - + stats, ap, ap_class = [], [], [] iouv = np.linspace(0.5, 0.95, 10) niou = iouv.size @@ -551,12 +668,11 @@ class MeanAveragePrecision: stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) continue if nl: - correct, iouv = cls.evaluate_detection_batch( + correct, iouv = cls._match_detection_batch( predictions=detection_batch, targets=true_batch, iouv=iouv, ) - # (correct, confidence, pred-class, target-class) stats.append( ( correct, @@ -569,14 +685,33 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - tp, fp, p, r, f1, ap, ap_class = MeanAveragePrecision.ap_per_class(*stats, names=classes) - ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 - mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() + p, r, f1, ap, ap_class = MeanAveragePrecision.ap_per_class(*stats) + ap50, ap75 = ap[:, 0], ap[:, 5] + ap = ap.mean(1) # AP@0.5, AP@0.5:0.95 + map50, map75, map = ap50.mean(), ap75.mean(), ap.mean() - return cls(map=map, map50=map50, ap50=ap50, per_class=ap_class, classes=classes) + return cls(map=map, map50=map50, map75=map75, per_class=ap_class, classes=classes) @staticmethod - def evaluate_detection_batch( + def detections_to_tensor( + detections: Detections, with_confidence: bool = False + ) -> np.ndarray: + + if detections.class_id is None: + raise ValueError( + "MeanAveragePrecision can only be calculated for Detections with class_id" + ) + + arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + + if with_confidence: + if detections.confidence is not None: + arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + + return np.concatenate(arrays_to_concat, axis=1) + + @staticmethod + def _match_detection_batch( predictions: np.ndarray, targets: np.ndarray, iouv: np.ndarray ) -> np.ndarray: correct = np.zeros((predictions.shape[0], iouv.shape[0])).astype(bool) @@ -623,6 +758,10 @@ class MeanAveragePrecision: @staticmethod def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: + """ + Deduplicate matches. If there are multiple matches for the same true or predicted box, + only the one with the highest IoU is kept. + """ if matches.shape[0] > 0: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] @@ -632,74 +771,73 @@ class MeanAveragePrecision: @staticmethod def compute_ap(recall, precision): - """Compute the average precision, given the recall and precision curves + """Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves # Arguments recall: The recall curve (list) precision: The precision curve (list) # Returns Average precision, precision curve, recall curve """ - # Append sentinel values to beginning and end mrec = np.concatenate(([0.0], recall, [1.0])) mpre = np.concatenate(([1.0], precision, [0.0])) - # Compute the precision envelope mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) - # Integrate area under curve - method = "interp" # methods: 'continuous', 'interp' - if method == "interp": - x = np.linspace(0, 1, 101) # 101-point interp (COCO) - ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate - else: # 'continuous' - i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes - ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve - - return ap, mpre, mrec + x = np.linspace(0, 1, 101) + ap = np.trapz(np.interp(x, mrec, mpre), x) + return ap @staticmethod - def ap_per_class(tp, conf, pred_cls, target_cls, eps=1e-16): - i = np.argsort(-conf) - tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + def ap_per_class(matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16): + """ + Args: + matches (np.ndarray): matches between predictions and targets + prediction_confidence (np.ndarray): confidence values of predictions + prediction_class_ids (np.ndarray): class ids values of predictions + true_batch_class_ids (np.ndarray): class ids values of targets + EPS: constant to avoid divide by zero + Returns: + precision, recall, f1_score, average_precisions, unique_classes + """ + sorted_ids = np.argsort(-prediction_confidence) + prediction_confidence = prediction_confidence[sorted_ids] + matches = matches[sorted_ids] - unique_classes, nt = np.unique(target_cls, return_counts=True) - nc = unique_classes.shape[0] # number of classes, number of detections + unique_classes, class_counts = np.unique(true_batch_class_ids, return_counts=True) + num_classes = unique_classes.shape[0] # number of classes, number of detections px, py = np.linspace(0, 1, 1000), [] # for plotting - ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + average_precisions = np.zeros((num_classes, matches.shape[1])) + precisions_all, recalls_all = np.zeros((num_classes, 1000)), np.zeros((num_classes, 1000)) for ci, c in enumerate(unique_classes): - i = pred_cls == c - n_l = nt[ci] # number of labels - n_p = i.sum() # number of predictions - if n_p == 0 or n_l == 0: + i = prediction_class_ids == c + num_targets = class_counts[ci] + num_predictions = i.sum() + if num_targets == 0 or num_predictions == 0: continue - # Accumulate FPs and TPs - fpc = (1 - tp[i]).cumsum(0) - tpc = tp[i].cumsum(0) + _false_positives = (1 - matches[i]).cumsum(0) + _true_positives = matches[i].cumsum(0) - # Recall - recall = tpc / (n_l + eps) # recall curve - r[ci] = np.interp( - -px, -conf[i], recall[:, 0], left=0 - ) # negative x, xp because xp decreases + recall = _true_positives / (num_targets + EPS) # recall curve + recalls_all[ci] = np.interp( + -px, -prediction_confidence[i], recall[:, 0], left=0 + ) - # Precision - precision = tpc / (tpc + fpc) # precision curve - p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score + precision = _true_positives / (_true_positives + _false_positives) # precision curve + precisions_all[ci] = np.interp(-px, -prediction_confidence[i], precision[:, 0], left=1) # p at pr_score # AP from recall-precision curve - for j in range(tp.shape[1]): - ap[ci, j], mpre, mrec = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) + for j in range(matches.shape[1]): + average_precisions[ci, j] = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) # Compute F1 (harmonic mean of precision and recall) - f1 = 2 * p * r / (p + r + eps) + f1_scores = 2 * precisions_all * recalls_all / (precisions_all + recalls_all + EPS) - i = MeanAveragePrecision.smooth(f1.mean(0), 0.1).argmax() # max F1 index - p, r, f1 = p[:, i], r[:, i], f1[:, i] - tp = (r * nt).round() # true positives - fp = (tp / (p + eps) - tp).round() # false positives - return tp, fp, p, r, f1, ap, unique_classes.astype(int) + i = MeanAveragePrecision.smooth(f1_scores.mean(0), 0.1).argmax() # max F1 index + precision, recall, f1_score = precisions_all[:, i], recalls_all[:, i], f1_scores[:, i] + + return precision, recall, f1_score, average_precisions, unique_classes.astype(int) @staticmethod def smooth(y, f=0.05): From 9b6b74dc77c1689b92ebb2ea3d91a7c766bd4217 Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 16:30:54 +0200 Subject: [PATCH 10/54] updated docstrings --- supervision/metrics/detection.py | 65 +++++++++----------------------- 1 file changed, 18 insertions(+), 47 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 4c34c5a4..c5a4ed81 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -469,17 +469,13 @@ class MeanAveragePrecision: map: float map50: float map75: float - per_class: List[np.ndarray] - classes: List[str] """ Mean Average Precision for object detection tasks. Attributes: map (float): map value. map50 (float): map value at iou threshold=0.5. - map75 (float): map value at iou threshold=0.75. - per_class (np.ndarray): numpy array of map values for each class - classes (List[str]): Model class names. + map75 (float): map value at iou threshold=0.75 """ @classmethod @@ -487,20 +483,15 @@ class MeanAveragePrecision: cls, predictions: List[Detections], targets: List[Detections], - classes: List[str], ) -> MeanAveragePrecision: """ - Calculate confusion matrix based on predicted and ground-truth detections. + Calculate MeanAveragePrecision based on predicted and ground-truth detections. Args: targets (List[Detections]): Detections objects from ground-truth. predictions (List[Detections]): Detections objects predicted by the model. - classes (List[str]): Model class names. - conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded. - iou_threshold (float): Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`. - Returns: - ConfusionMatrix: New instance of ConfusionMatrix. + MeanAveragePrecision: New instance of ConfusionMatrix. Example: ```python @@ -519,7 +510,6 @@ class MeanAveragePrecision: >>> mean_average_precison = sv.MeanAveragePrecision.from_detections( ... predictions=predictions, ... targets=target, - ... classes=['person', ...] ... ) >>> mean_average_precison.matrix @@ -540,7 +530,6 @@ class MeanAveragePrecision: return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, - classes=classes, ) @classmethod @@ -588,7 +577,6 @@ class MeanAveragePrecision: return cls.from_detections( predictions=predictions, targets=targets, - classes=dataset.classes, ) @classmethod @@ -596,7 +584,6 @@ class MeanAveragePrecision: cls, predictions: List[np.ndarray], targets: List[np.ndarray], - classes: List[str], ) -> MeanAveragePrecision: """ Calculate Mean Average Precision based on predicted and ground-truth detections at different threshold. @@ -604,7 +591,6 @@ class MeanAveragePrecision: Args: predictions (List[np.ndarray]): Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format. targets (List[np.ndarray]): Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format. - classes (List[str]): Model class names. Returns: MeanAveragePrecision: New instance of MeanAveragePrecision. @@ -640,7 +626,6 @@ class MeanAveragePrecision: >>> mean_average_precison = sv.MeanAveragePrecision.from_tensors( ... predictions=predictions, ... targets=targets, - ... classes=['person', ...] ... ) >>> mean_average_precison.map @@ -648,20 +633,18 @@ class MeanAveragePrecision: ``` """ cls._validate_input_tensors(predictions, targets) - map, map50, map75, map90 = 0, 0, 0, 0 - ap50, ap75, ap90 = 0, 0, 0 - p, r, f1 = 0, 0, 0 + map, map50, map75 = 0, 0, 0 - stats, ap, ap_class = [], [], [] - iouv = np.linspace(0.5, 0.95, 10) - niou = iouv.size + stats, ap = [], [] + iou_levels = np.linspace(0.5, 0.95, 10) + num_ious = iou_levels.size for true_batch, detection_batch in zip(targets, predictions): nl, npr = ( true_batch.shape[0], detection_batch.shape[0], ) # number of labels, predictions - correct = np.zeros((npr, niou), dtype=bool) # init + correct = np.zeros((npr, num_ious), dtype=bool) # init if npr == 0: if nl: @@ -671,7 +654,7 @@ class MeanAveragePrecision: correct, iouv = cls._match_detection_batch( predictions=detection_batch, targets=true_batch, - iouv=iouv, + iou_levels=iou_levels, ) stats.append( ( @@ -685,12 +668,12 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - p, r, f1, ap, ap_class = MeanAveragePrecision.ap_per_class(*stats) + ap = MeanAveragePrecision.ap_per_class(*stats) ap50, ap75 = ap[:, 0], ap[:, 5] ap = ap.mean(1) # AP@0.5, AP@0.5:0.95 map50, map75, map = ap50.mean(), ap75.mean(), ap.mean() - return cls(map=map, map50=map50, map75=map75, per_class=ap_class, classes=classes) + return cls(map=map, map50=map50, map75=map75) @staticmethod def detections_to_tensor( @@ -712,14 +695,14 @@ class MeanAveragePrecision: @staticmethod def _match_detection_batch( - predictions: np.ndarray, targets: np.ndarray, iouv: np.ndarray + predictions: np.ndarray, targets: np.ndarray, iou_levels: np.ndarray ) -> np.ndarray: - correct = np.zeros((predictions.shape[0], iouv.shape[0])).astype(bool) + correct = np.zeros((predictions.shape[0], iou_levels.shape[0])).astype(bool) iou = box_iou_batch(targets[:, :4], predictions[:, :4]) correct_class = targets[:, 4:5] == predictions[:, 4] - for i in range(len(iouv)): - x = np.where((iou >= iouv[i]) & correct_class) + for i in range(len(iou_levels)): + x = np.where((iou >= iou_levels[i]) & correct_class) if x[0].shape[0]: _X1 = np.vstack([x[0], x[1]]).T @@ -727,7 +710,7 @@ class MeanAveragePrecision: matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] matches = MeanAveragePrecision._drop_extra_matches(matches) correct[matches[:, 1].astype(int), i] = True - return correct, iouv + return correct, iou_levels @classmethod def _validate_input_tensors( @@ -800,15 +783,13 @@ class MeanAveragePrecision: precision, recall, f1_score, average_precisions, unique_classes """ sorted_ids = np.argsort(-prediction_confidence) - prediction_confidence = prediction_confidence[sorted_ids] matches = matches[sorted_ids] unique_classes, class_counts = np.unique(true_batch_class_ids, return_counts=True) num_classes = unique_classes.shape[0] # number of classes, number of detections - px, py = np.linspace(0, 1, 1000), [] # for plotting average_precisions = np.zeros((num_classes, matches.shape[1])) - precisions_all, recalls_all = np.zeros((num_classes, 1000)), np.zeros((num_classes, 1000)) + for ci, c in enumerate(unique_classes): i = prediction_class_ids == c num_targets = class_counts[ci] @@ -820,24 +801,14 @@ class MeanAveragePrecision: _true_positives = matches[i].cumsum(0) recall = _true_positives / (num_targets + EPS) # recall curve - recalls_all[ci] = np.interp( - -px, -prediction_confidence[i], recall[:, 0], left=0 - ) precision = _true_positives / (_true_positives + _false_positives) # precision curve - precisions_all[ci] = np.interp(-px, -prediction_confidence[i], precision[:, 0], left=1) # p at pr_score # AP from recall-precision curve for j in range(matches.shape[1]): average_precisions[ci, j] = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) - # Compute F1 (harmonic mean of precision and recall) - f1_scores = 2 * precisions_all * recalls_all / (precisions_all + recalls_all + EPS) - - i = MeanAveragePrecision.smooth(f1_scores.mean(0), 0.1).argmax() # max F1 index - precision, recall, f1_score = precisions_all[:, i], recalls_all[:, i], f1_scores[:, i] - - return precision, recall, f1_score, average_precisions, unique_classes.astype(int) + return average_precisions @staticmethod def smooth(y, f=0.05): From e9fbcf5dcb712763d164a68b687e210394132cb8 Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 16:33:10 +0200 Subject: [PATCH 11/54] updated docstrings --- docs/metrics/detection.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/metrics/detection.md b/docs/metrics/detection.md index 9e784464..feea7cce 100644 --- a/docs/metrics/detection.md +++ b/docs/metrics/detection.md @@ -6,3 +6,7 @@ ## ConfusionMatrix :::supervision.metrics.detection.ConfusionMatrix + +## MeanAveragePrecision + +:::supervision.metrics.detection.MeanAveragePrecision From 44a174f889d323be6d89e7bb93257b57c901648d Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 16:33:42 +0200 Subject: [PATCH 12/54] removed extra blank lines --- supervision/metrics/detection.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index c5a4ed81..36fc515a 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -817,6 +817,3 @@ class MeanAveragePrecision: p = np.ones(nf // 2) # ones padding yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed - - - From 0754cb75bda1d008fbcc7faa0a95b66f22305b77 Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 17:53:19 +0200 Subject: [PATCH 13/54] update --- supervision/metrics/detection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 36fc515a..428e441e 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -669,8 +669,8 @@ class MeanAveragePrecision: if len(stats) and stats[0].any(): ap = MeanAveragePrecision.ap_per_class(*stats) - ap50, ap75 = ap[:, 0], ap[:, 5] - ap = ap.mean(1) # AP@0.5, AP@0.5:0.95 + + ap50, ap75, ap = ap[:, 0], ap[:, 5], ap.mean(1) # AP@0.5, AP@0.5:0.95 map50, map75, map = ap50.mean(), ap75.mean(), ap.mean() return cls(map=map, map50=map50, map75=map75) @@ -699,6 +699,7 @@ class MeanAveragePrecision: ) -> np.ndarray: correct = np.zeros((predictions.shape[0], iou_levels.shape[0])).astype(bool) iou = box_iou_batch(targets[:, :4], predictions[:, :4]) + correct_class = targets[:, 4:5] == predictions[:, 4] for i in range(len(iou_levels)): From f4dc515df1c4a90fc05ebe0d2d4d8c4222e42e28 Mon Sep 17 00:00:00 2001 From: hardik Date: Mon, 24 Jul 2023 18:20:01 +0200 Subject: [PATCH 14/54] update --- supervision/metrics/detection.py | 51 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 428e441e..dd6b3b92 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -651,7 +651,7 @@ class MeanAveragePrecision: stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) continue if nl: - correct, iouv = cls._match_detection_batch( + correct, iou_levels = cls._match_detection_batch( predictions=detection_batch, targets=true_batch, iou_levels=iou_levels, @@ -668,8 +668,7 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - ap = MeanAveragePrecision.ap_per_class(*stats) - + ap = cls.ap_per_class(*stats) ap50, ap75, ap = ap[:, 0], ap[:, 5], ap.mean(1) # AP@0.5, AP@0.5:0.95 map50, map75, map = ap50.mean(), ap75.mean(), ap.mean() @@ -773,39 +772,41 @@ class MeanAveragePrecision: @staticmethod def ap_per_class(matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + # Arguments + tp: True positives (nparray, nx1 or nx10). + conf: Objectness value from 0-1 (nparray). + pred_cls: Predicted object classes (nparray). + target_cls: True object classes (nparray). + plot: Plot precision-recall curve at mAP@0.5 + save_dir: Plot save directory + # Returns + The average precision as computed in py-faster-rcnn. """ - Args: - matches (np.ndarray): matches between predictions and targets - prediction_confidence (np.ndarray): confidence values of predictions - prediction_class_ids (np.ndarray): class ids values of predictions - true_batch_class_ids (np.ndarray): class ids values of targets - EPS: constant to avoid divide by zero - Returns: - precision, recall, f1_score, average_precisions, unique_classes - """ - sorted_ids = np.argsort(-prediction_confidence) - matches = matches[sorted_ids] + sorted_confidences = np.argsort(-prediction_confidence) + matches = matches[sorted_confidences] + prediction_class_ids = prediction_class_ids[sorted_confidences] + + # Find unique classes unique_classes, class_counts = np.unique(true_batch_class_ids, return_counts=True) num_classes = unique_classes.shape[0] # number of classes, number of detections average_precisions = np.zeros((num_classes, matches.shape[1])) - for ci, c in enumerate(unique_classes): - i = prediction_class_ids == c - num_targets = class_counts[ci] - num_predictions = i.sum() - if num_targets == 0 or num_predictions == 0: + valid = prediction_class_ids == c + num_targets = class_counts[ci] # number of labels + num_predictions = valid.sum() # number of predictions + if num_predictions == 0 or num_targets == 0: continue - _false_positives = (1 - matches[i]).cumsum(0) - _true_positives = matches[i].cumsum(0) + fp_pool = (1 - matches[valid]).cumsum(0) + tp_pool = matches[valid].cumsum(0) - recall = _true_positives / (num_targets + EPS) # recall curve + recall = tp_pool / (num_targets + EPS) + precision = tp_pool / (tp_pool + fp_pool) - precision = _true_positives / (_true_positives + _false_positives) # precision curve - - # AP from recall-precision curve for j in range(matches.shape[1]): average_precisions[ci, j] = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) From da6a324c1f9f2ed06b8ef683663dc4b3be03c9ea Mon Sep 17 00:00:00 2001 From: hardik Date: Tue, 25 Jul 2023 14:32:19 +0200 Subject: [PATCH 15/54] update --- supervision/metrics/detection.py | 79 +++++++++++++------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 7d35df42..6b0ad247 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -651,7 +651,7 @@ class MeanAveragePrecision: stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) continue if nl: - correct, iou_levels = cls._match_detection_batch( + correct = cls._match_detection_batch( predictions=detection_batch, targets=true_batch, iou_levels=iou_levels, @@ -705,39 +705,12 @@ class MeanAveragePrecision: x = np.where((iou >= iou_levels[i]) & correct_class) if x[0].shape[0]: - _X1 = np.vstack([x[0], x[1]]).T + _X1 = np.concatenate([np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1) _x2 = iou[x[0], x[1]][:, None] matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] matches = MeanAveragePrecision._drop_extra_matches(matches) correct[matches[:, 1].astype(int), i] = True - return correct, iou_levels - - @classmethod - def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] - ): - """ - Checks for shape consistency of input tensors. - """ - if len(predictions) != len(targets): - raise ValueError( - f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." - ) - if len(predictions) > 0: - if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray - ): - raise ValueError( - f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." - ) - if predictions[0].shape[1] != 6: - raise ValueError( - f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." - ) - if targets[0].shape[1] != 5: - raise ValueError( - f"Targets must have shape (N, 5). Got {targets[0].shape} instead." - ) + return correct @staticmethod def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: @@ -775,16 +748,11 @@ class MeanAveragePrecision: """ Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. # Arguments - tp: True positives (nparray, nx1 or nx10). - conf: Objectness value from 0-1 (nparray). - pred_cls: Predicted object classes (nparray). - target_cls: True object classes (nparray). - plot: Plot precision-recall curve at mAP@0.5 - save_dir: Plot save directory - # Returns - The average precision as computed in py-faster-rcnn. + matches: True positives (nparray, nx1 or nx10). + prediction_confidence: Objectness value from 0-1 (nparray). + prediction_class_ids: Predicted object classes (nparray). + true_batch_class_ids: True object classes (nparray). """ - sorted_confidences = np.argsort(-prediction_confidence) matches = matches[sorted_confidences] prediction_class_ids = prediction_class_ids[sorted_confidences] @@ -812,10 +780,29 @@ class MeanAveragePrecision: return average_precisions - @staticmethod - def smooth(y, f=0.05): - # Box filter of fraction f - nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) - p = np.ones(nf // 2) # ones padding - yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded - return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed + @classmethod + def _validate_input_tensors( + cls, predictions: List[np.ndarray], targets: List[np.ndarray] + ): + """ + Checks for shape consistency of input tensors. + """ + if len(predictions) != len(targets): + raise ValueError( + f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." + ) + if len(predictions) > 0: + if not isinstance(predictions[0], np.ndarray) or not isinstance( + targets[0], np.ndarray + ): + raise ValueError( + f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." + ) + if predictions[0].shape[1] != 6: + raise ValueError( + f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." + ) + if targets[0].shape[1] != 5: + raise ValueError( + f"Targets must have shape (N, 5). Got {targets[0].shape} instead." + ) From dec982200b12c7e738636991ea4e44e04da5a12b Mon Sep 17 00:00:00 2001 From: hardik Date: Tue, 25 Jul 2023 17:53:07 +0200 Subject: [PATCH 16/54] update --- supervision/metrics/detection.py | 53 +++++++++++++++----------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 6b0ad247..95f35f1d 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -520,12 +520,10 @@ class MeanAveragePrecision: target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - MeanAveragePrecision.detections_to_tensor( - prediction, with_confidence=True - ) + MeanAveragePrecision.detections_to_tensor(prediction) ) target_tensors.append( - MeanAveragePrecision.detections_to_tensor(target, with_confidence=False) + MeanAveragePrecision.targets_to_tensor(target) ) return cls.from_tensors( predictions=prediction_tensors, @@ -635,6 +633,9 @@ class MeanAveragePrecision: cls._validate_input_tensors(predictions, targets) map, map50, map75 = 0, 0, 0 + class_index = 4 + conf_index = 5 + stats, ap = [], [] iou_levels = np.linspace(0.5, 0.95, 10) num_ious = iou_levels.size @@ -643,7 +644,7 @@ class MeanAveragePrecision: nl, npr = ( true_batch.shape[0], detection_batch.shape[0], - ) # number of labels, predictions + ) correct = np.zeros((npr, num_ious), dtype=bool) # init if npr == 0: @@ -659,9 +660,9 @@ class MeanAveragePrecision: stats.append( ( correct, - detection_batch[:, 5], - detection_batch[:, 4], - true_batch[:, 4], + detection_batch[:, conf_index], + detection_batch[:, class_index], + true_batch[:, class_index], ) ) @@ -676,21 +677,24 @@ class MeanAveragePrecision: @staticmethod def detections_to_tensor( - detections: Detections, with_confidence: bool = False + detections: Detections ) -> np.ndarray: - if detections.class_id is None: raise ValueError( "MeanAveragePrecision can only be calculated for Detections with class_id" ) - arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + return np.concatenate([detections.xyxy, np.expand_dims(detections.class_id, 1), np.expand_dims(detections.confidence, 1)], 1) - if with_confidence: - if detections.confidence is not None: - arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + @staticmethod + def targets_to_tensor( + detections: Detections) -> np.ndarray: - return np.concatenate(arrays_to_concat, axis=1) + if detections.class_id is None: + raise ValueError( + "MeanAveragePrecision can only be calculated for Detections with class_id" + ) + return np.hstack([detections.xyxy, np.expand_dims(detections.class_id, 1)]) @staticmethod def _match_detection_batch( @@ -708,23 +712,14 @@ class MeanAveragePrecision: _X1 = np.concatenate([np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1) _x2 = iou[x[0], x[1]][:, None] matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] - matches = MeanAveragePrecision._drop_extra_matches(matches) + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True correct[matches[:, 1].astype(int), i] = True return correct - @staticmethod - def _drop_extra_matches(matches: np.ndarray) -> np.ndarray: - """ - Deduplicate matches. If there are multiple matches for the same true or predicted box, - only the one with the highest IoU is kept. - """ - if matches.shape[0] > 0: - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 1], return_index=True)[1]] - matches = matches[matches[:, 2].argsort()[::-1]] - matches = matches[np.unique(matches[:, 0], return_index=True)[1]] - return matches - @staticmethod def compute_ap(recall, precision): """Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves From 8d8703a1c658a2dc0dcf9de35a8eb7327231beb5 Mon Sep 17 00:00:00 2001 From: hardik Date: Tue, 25 Jul 2023 18:15:26 +0200 Subject: [PATCH 17/54] update --- supervision/metrics/detection.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 95f35f1d..b7985c64 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Callable, List, Optional, Tuple - +import json import matplotlib import matplotlib.pyplot as plt import numpy as np @@ -469,6 +469,7 @@ class MeanAveragePrecision: map: float map50: float map75: float + average_precisions: np.ndarray """ Mean Average Precision for object detection tasks. @@ -636,7 +637,7 @@ class MeanAveragePrecision: class_index = 4 conf_index = 5 - stats, ap = [], [] + stats, average_precisions = [], [] iou_levels = np.linspace(0.5, 0.95, 10) num_ious = iou_levels.size @@ -669,11 +670,11 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - ap = cls.ap_per_class(*stats) - ap50, ap75, ap = ap[:, 0], ap[:, 5], ap.mean(1) # AP@0.5, AP@0.5:0.95 - map50, map75, map = ap50.mean(), ap75.mean(), ap.mean() + average_precisions = cls.average_precisions_per_class(*stats) + ap50, ap75, average_precisions = average_precisions[:, 0], average_precisions[:, 5], average_precisions.mean(1) + map50, map75, map = ap50.mean(), ap75.mean(), average_precisions.mean() - return cls(map=map, map50=map50, map75=map75) + return cls(map=map, map50=map50, map75=map75, average_precisions=average_precisions) @staticmethod def detections_to_tensor( @@ -721,7 +722,7 @@ class MeanAveragePrecision: return correct @staticmethod - def compute_ap(recall, precision): + def compute_average_precision(recall, precision): """Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves # Arguments recall: The recall curve (list) @@ -739,7 +740,7 @@ class MeanAveragePrecision: return ap @staticmethod - def ap_per_class(matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16): + def average_precisions_per_class(matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16): """ Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. # Arguments @@ -771,7 +772,7 @@ class MeanAveragePrecision: precision = tp_pool / (tp_pool + fp_pool) for j in range(matches.shape[1]): - average_precisions[ci, j] = MeanAveragePrecision.compute_ap(recall[:, j], precision[:, j]) + average_precisions[ci, j] = MeanAveragePrecision.compute_average_precision(recall[:, j], precision[:, j]) return average_precisions @@ -801,3 +802,6 @@ class MeanAveragePrecision: raise ValueError( f"Targets must have shape (N, 5). Got {targets[0].shape} instead." ) + + def to_dict(self): + return {'map': self.map, 'map50': self.map50, 'map75': self.map75, 'average_precisions': self.average_precisions} \ No newline at end of file From d7e08d42eb0ebf0acf47bbdbb07a7342e43140b7 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 10:20:47 +0200 Subject: [PATCH 18/54] add load_pascal_voc_annotations v2 --- supervision/dataset/formats/pascal_voc.py | 93 ++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index a3005eb9..bca886d4 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -1,12 +1,16 @@ -from typing import List, Optional, Tuple +import os +from pathlib import Path +from typing import Dict, List, Optional, Tuple from xml.dom.minidom import parseString from xml.etree.ElementTree import Element, SubElement, parse, tostring +import cv2 import numpy as np from supervision.dataset.utils import approximate_mask_with_polygons from supervision.detection.core import Detections from supervision.detection.utils import polygon_to_xyxy +from supervision.utils.file import list_files_with_extensions def object_to_pascal_voc( @@ -120,6 +124,93 @@ def detections_to_pascal_voc( def load_pascal_voc_annotations( + images_directory_path: str, + annotations_directory_path: str, + force_masks: bool = False, +) -> Tuple[List[str], Dict[str, np.ndarray], Dict[str, Detections]]: + """ + Loads PASCAL VOC XML annotations and returns the image name, a Detections instance, and a list of class names. + + Args: + annotation_path (str): The path to the PASCAL VOC XML annotations file. + + Returns: + Tuple[str, Detections, List[str]]: A tuple containing the image name, a Detections instance, and a list of class names of objects in the detections. + """ + + image_paths = list_files_with_extensions( + directory=images_directory_path, extensions=["jpg", "jpeg", "png"] + ) + + classes = [] + images = {} + annotations = {} + + for image_path in image_paths: + image_name = Path(image_path).stem + image = cv2.imread(str(image_path)) + + annotation_path = os.path.join(annotations_directory_path, f"{image_name}.xml") + if not os.path.exists(annotation_path): + images[image_path.name] = image + annotations[image_path.name] = Detections.empty() + continue + + tree = parse(annotation_path) + root = tree.getroot() + + xyxy = [] + class_names = [] + masks = [] + for obj in root.findall("object"): + class_name = obj.find("name").text + class_names.append(class_name) + + bbox = obj.find("bndbox") + x1 = int(bbox.find("xmin").text) + y1 = int(bbox.find("ymin").text) + x2 = int(bbox.find("xmax").text) + y2 = int(bbox.find("ymax").text) + + xyxy.append([x1, y1, x2, y2]) + + with_masks = obj.find("polygon") is not None + with_masks = force_masks if force_masks else with_masks + + for polygon in obj.findall("polygon"): + polygon_points = [] + coords = polygon.findall(".//*") + for i in range(0, len(coords), 2): + x = int(coords[i].text) + y = int(coords[i + 1].text) + polygon_points.append([x, y]) + + mask_from_polygon = polygon_to_mask( + polygon=np.array(polygon_points), + resolution_wh=(image.shape[0], image.shape[1]), + ) + masks.append(mask_from_polygon) + + xyxy = np.array(xyxy) + masks = np.array(masks) + annotation = Detections(xyxy=xyxy, mask=masks, class_id=np.array(class_names)) + + images[image_path.name] = image + annotations[image_path.name] = annotation + classes += class_names + + classes = list(set(classes)) + + return classes, images, annotations + + +def polygon_to_mask(polygon: np.ndarray, resolution_wh: Tuple[int, int]) -> np.ndarray: + mask = np.zeros(resolution_wh, dtype=np.uint8) + cv2.fillPoly(mask, pts=[polygon], color=1) + return mask + + +def load_pascal_voc_annotations_v1( annotation_path: str, ) -> Tuple[str, Detections, List[str]]: """ From 3723eae50c385fc2cbbf4e3cd97b68d5056e28ef Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 10:21:17 +0200 Subject: [PATCH 19/54] update from_pascal_voc to match v2 loader --- supervision/dataset/core.py | 38 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index ce7e0edb..4bd80c1b 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -197,7 +197,10 @@ class DetectionDataset(BaseDataset): @classmethod def from_pascal_voc( - cls, images_directory_path: str, annotations_directory_path: str + cls, + images_directory_path: str, + annotations_directory_path: str, + force_masks: bool = False, ) -> DetectionDataset: """ Creates a Dataset instance from PASCAL VOC formatted data. @@ -231,34 +234,17 @@ class DetectionDataset(BaseDataset): ['dog', 'person'] ``` """ - image_paths = list_files_with_extensions( - directory=images_directory_path, extensions=["jpg", "jpeg", "png"] - ) - annotation_paths = list_files_with_extensions( - directory=annotations_directory_path, extensions=["xml"] + + classes, images, annotations = load_pascal_voc_annotations( + images_directory_path=images_directory_path, + annotations_directory_path=annotations_directory_path, + force_masks=force_masks, ) - raw_annotations: List[Tuple[str, Detections, List[str]]] = [ - load_pascal_voc_annotations(annotation_path=str(annotation_path)) - for annotation_path in annotation_paths - ] + for annotation in annotations.values(): + class_id = [classes.index(class_name) for class_name in annotation.class_id] + annotation.class_id = np.array(class_id) - classes = [] - for annotation in raw_annotations: - classes.extend(annotation[2]) - classes = list(set(classes)) - - for annotation in raw_annotations: - class_id = [classes.index(class_name) for class_name in annotation[2]] - annotation[1].class_id = np.array(class_id) - - images = { - image_path.name: cv2.imread(str(image_path)) for image_path in image_paths - } - - annotations = { - image_name: detections for image_name, detections, _ in raw_annotations - } return DetectionDataset(classes=classes, images=images, annotations=annotations) @classmethod From 86b33ed55801d33d1c14dd89d1a95f6784228d83 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 10:39:45 +0200 Subject: [PATCH 20/54] add test_pascal template --- test/dataset/formats/test_pascal.py | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/dataset/formats/test_pascal.py diff --git a/test/dataset/formats/test_pascal.py b/test/dataset/formats/test_pascal.py new file mode 100644 index 00000000..30bbc05e --- /dev/null +++ b/test/dataset/formats/test_pascal.py @@ -0,0 +1,32 @@ +from contextlib import ExitStack as DoesNotRaise +from typing import List, Optional, Tuple + +import numpy as np +import pytest + +from supervision.dataset.formats.pascal_voc import ( + detections_to_pascal_voc, + load_pascal_voc_annotations, + object_to_pascal_voc, +) +from supervision.detection.core import Detections + +# TODO + + +def test_detections_to_pascal_voc( + expected_result, exception: Exception +): + ... + + +def test_load_pascal_voc_annotations( + expected_result, exception: Exception +): + ... + + +def test_object_to_pascal_voc( + expected_result, exception: Exception +): + ... From a91118322b84ebf9c01b1b94a410490c18983e35 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 10:45:46 +0200 Subject: [PATCH 21/54] update docstrings --- supervision/dataset/core.py | 2 +- supervision/dataset/formats/pascal_voc.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index 4bd80c1b..d40728e4 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -225,7 +225,7 @@ class DetectionDataset(BaseDataset): >>> project = rf.workspace(WORKSPACE_ID).project(PROJECT_ID) >>> dataset = project.version(PROJECT_VERSION).download("voc") - >>> ds = sv.DetectionDataset.from_yolo( + >>> ds = sv.DetectionDataset.from_pascal_voc( ... images_directory_path=f"{dataset.location}/train/images", ... annotations_directory_path=f"{dataset.location}/train/labels" ... ) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index bca886d4..90501be3 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -129,13 +129,15 @@ def load_pascal_voc_annotations( force_masks: bool = False, ) -> Tuple[List[str], Dict[str, np.ndarray], Dict[str, Detections]]: """ - Loads PASCAL VOC XML annotations and returns the image name, a Detections instance, and a list of class names. + Loads PASCAL VOC annotations and returns class names, images, and their corresponding detections. Args: - annotation_path (str): The path to the PASCAL VOC XML annotations file. + images_directory_path (str): The path to the directory containing the images. + annotations_directory_path (str): The path to the directory containing the PASCAL VOC annotation files. + force_masks (bool, optional): If True, forces masks to be loaded for all annotations, regardless of whether they are present. Returns: - Tuple[str, Detections, List[str]]: A tuple containing the image name, a Detections instance, and a list of class names of objects in the detections. + Tuple[List[str], Dict[str, np.ndarray], Dict[str, Detections]]: A tuple containing a list of class names, a dictionary with image names as keys and images as values, and a dictionary with image names as keys and corresponding Detections instances as values. """ image_paths = list_files_with_extensions( From 6dde70eb0c7ffcfa97191b9276bbbc77c82471ce Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 10:46:31 +0200 Subject: [PATCH 22/54] move fixing of class_ids to load_pascal_voc_annotations --- supervision/dataset/core.py | 4 ---- supervision/dataset/formats/pascal_voc.py | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index d40728e4..15d92aa0 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -241,10 +241,6 @@ class DetectionDataset(BaseDataset): force_masks=force_masks, ) - for annotation in annotations.values(): - class_id = [classes.index(class_name) for class_name in annotation.class_id] - annotation.class_id = np.array(class_id) - return DetectionDataset(classes=classes, images=images, annotations=annotations) @classmethod diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 90501be3..8f3fb090 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -203,6 +203,10 @@ def load_pascal_voc_annotations( classes = list(set(classes)) + for annotation in annotations.values(): + class_id = [classes.index(class_name) for class_name in annotation.class_id] + annotation.class_id = np.array(class_id) + return classes, images, annotations From 277a9cb6b9d3d073b1232e167b3edfddebeee696 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Wed, 26 Jul 2023 12:36:49 +0200 Subject: [PATCH 23/54] import polygon_to_mask from supervision --- supervision/dataset/formats/pascal_voc.py | 8 +------- .../formats/{test_pascal.py => test_pascal_voc.py} | 0 2 files changed, 1 insertion(+), 7 deletions(-) rename test/dataset/formats/{test_pascal.py => test_pascal_voc.py} (100%) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 8f3fb090..8cc3cef2 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -9,7 +9,7 @@ import numpy as np from supervision.dataset.utils import approximate_mask_with_polygons from supervision.detection.core import Detections -from supervision.detection.utils import polygon_to_xyxy +from supervision.detection.utils import polygon_to_mask, polygon_to_xyxy from supervision.utils.file import list_files_with_extensions @@ -210,12 +210,6 @@ def load_pascal_voc_annotations( return classes, images, annotations -def polygon_to_mask(polygon: np.ndarray, resolution_wh: Tuple[int, int]) -> np.ndarray: - mask = np.zeros(resolution_wh, dtype=np.uint8) - cv2.fillPoly(mask, pts=[polygon], color=1) - return mask - - def load_pascal_voc_annotations_v1( annotation_path: str, ) -> Tuple[str, Detections, List[str]]: diff --git a/test/dataset/formats/test_pascal.py b/test/dataset/formats/test_pascal_voc.py similarity index 100% rename from test/dataset/formats/test_pascal.py rename to test/dataset/formats/test_pascal_voc.py From 4b621c3c8d76e091dc4db0724175f14e4c9063a7 Mon Sep 17 00:00:00 2001 From: hardik Date: Wed, 26 Jul 2023 13:58:53 +0200 Subject: [PATCH 24/54] update --- supervision/metrics/detection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index b7985c64..15c1663f 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -2,7 +2,7 @@ from __future__ import annotations from dataclasses import dataclass from typing import Callable, List, Optional, Tuple -import json + import matplotlib import matplotlib.pyplot as plt import numpy as np @@ -514,7 +514,7 @@ class MeanAveragePrecision: ... ) >>> mean_average_precison.matrix - 0.433 + 0.2899 ``` """ prediction_tensors = [] @@ -628,7 +628,7 @@ class MeanAveragePrecision: ... ) >>> mean_average_precison.map - 0.433 + 0.2899 ``` """ cls._validate_input_tensors(predictions, targets) From 70fe357a4edd4b321436a45bbcd49f0431b66ae1 Mon Sep 17 00:00:00 2001 From: hardik Date: Wed, 26 Jul 2023 18:13:24 +0200 Subject: [PATCH 25/54] fixed docstring --- supervision/metrics/detection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 15c1663f..bb5e6f0e 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -477,6 +477,7 @@ class MeanAveragePrecision: map (float): map value. map50 (float): map value at iou threshold=0.5. map75 (float): map value at iou threshold=0.75 + average_precisions (np.ndarray): values for every classes """ @classmethod From b0e0d074af503ad8c7d35276ebf99b8ce5fe24d1 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Fri, 28 Jul 2023 22:21:47 +0200 Subject: [PATCH 26/54] fixes patch --- supervision/dataset/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index 15d92aa0..6e47d747 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -31,7 +31,6 @@ from supervision.dataset.utils import ( train_test_split, ) from supervision.detection.core import Detections -from supervision.utils.file import list_files_with_extensions @dataclass @@ -208,6 +207,7 @@ class DetectionDataset(BaseDataset): Args: images_directory_path (str): The path to the directory containing the images. annotations_directory_path (str): The path to the directory containing the PASCAL VOC XML annotations. + force_masks (bool, optional): If True, forces masks to be loaded for all annotations, regardless of whether they are present. Returns: DetectionDataset: A DetectionDataset instance containing the loaded images and annotations. From 0a1212f23d532aecf01a99a44e8dec079a25d473 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Fri, 28 Jul 2023 22:22:03 +0200 Subject: [PATCH 27/54] fix with_masks defined but not used --- supervision/dataset/formats/pascal_voc.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 8cc3cef2..71a929a6 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -164,6 +164,7 @@ def load_pascal_voc_annotations( xyxy = [] class_names = [] masks = [] + with_masks = False for obj in root.findall("object"): class_name = obj.find("name").text class_names.append(class_name) @@ -194,8 +195,14 @@ def load_pascal_voc_annotations( masks.append(mask_from_polygon) xyxy = np.array(xyxy) - masks = np.array(masks) - annotation = Detections(xyxy=xyxy, mask=masks, class_id=np.array(class_names)) + + if with_masks: + masks = np.array(masks) + annotation = Detections( + xyxy=xyxy, mask=masks, class_id=np.array(class_names) + ) + else: + annotation = Detections(xyxy=xyxy, class_id=np.array(class_names)) images[image_path.name] = image annotations[image_path.name] = annotation From a181826527d7929604c7f3348f60be0e99a9fd9d Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Fri, 28 Jul 2023 23:13:11 +0200 Subject: [PATCH 28/54] add test_object_to_pascal_voc --- test/dataset/formats/test_pascal_voc.py | 73 ++++++++++++++++++++----- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index 30bbc05e..f5a953a2 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -1,4 +1,6 @@ +import xml.etree.ElementTree as ET from contextlib import ExitStack as DoesNotRaise +from test.utils import mock_detections from typing import List, Optional, Tuple import numpy as np @@ -11,22 +13,65 @@ from supervision.dataset.formats.pascal_voc import ( ) from supervision.detection.core import Detections -# TODO - - -def test_detections_to_pascal_voc( - expected_result, exception: Exception -): - ... - - -def test_load_pascal_voc_annotations( - expected_result, exception: Exception -): - ... + +def are_xml_elements_equal(elem1, elem2): + if ( + elem1.tag != elem2.tag + or elem1.attrib != elem2.attrib + or elem1.text != elem2.text + or len(elem1) != len(elem2) + ): + return False + + for child1, child2 in zip(elem1, elem2): + if not are_xml_elements_equal(child1, child2): + return False + + return True +@pytest.mark.parametrize( + "xyxy, name, polygon, expected_result, exception", + [ + ( + [0, 0, 10, 10], + "test", + None, + ET.fromstring( + """test001010""" + ), + DoesNotRaise(), + ), + ( + [0, 0, 10, 10], + "test", + [[0, 0], [10, 0], [10, 10], [0, 10]], + ET.fromstring( + """test001010001001010010""" + ), + DoesNotRaise(), + ), + ], +) def test_object_to_pascal_voc( - expected_result, exception: Exception + xyxy: np.ndarray, + name: str, + polygon: Optional[np.ndarray], + expected_result, + exception: Exception, ): + with exception: + result = object_to_pascal_voc(xyxy=xyxy, name=name, polygon=polygon) + with open("/tmp/test.xml", "w") as f: + f.write(ET.tostring(result).decode()) + with open("/tmp/exptest.xml", "w") as f: + f.write(ET.tostring(expected_result).decode()) + assert are_xml_elements_equal(result, expected_result) + + +def test_load_pascal_voc_annotations(expected_result, exception: Exception): + ... + + +def test_detections_to_pascal_voc(expected_result, exception: Exception): ... From 446ae23d99a7eeadd9497806f70e62bd9d037d9b Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 00:38:08 +0200 Subject: [PATCH 29/54] fix registering of empty detection --- supervision/dataset/formats/pascal_voc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 71a929a6..9dc15bbb 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -194,7 +194,7 @@ def load_pascal_voc_annotations( ) masks.append(mask_from_polygon) - xyxy = np.array(xyxy) + xyxy = np.array(xyxy) if len(xyxy) > 0 else np.empty((0, 4)) if with_masks: masks = np.array(masks) From 9a9d0c5259cda8961e0346ccbdb10ce7a54880e7 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 00:39:11 +0200 Subject: [PATCH 30/54] refactor class_id assignment to Detections in VOC --- supervision/dataset/formats/pascal_voc.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 9dc15bbb..ad78246b 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -195,24 +195,18 @@ def load_pascal_voc_annotations( masks.append(mask_from_polygon) xyxy = np.array(xyxy) if len(xyxy) > 0 else np.empty((0, 4)) + for k in set(class_names): + if k not in classes: + classes.append(k) + class_id = np.array([classes.index(class_name) for class_name in class_names]) if with_masks: - masks = np.array(masks) - annotation = Detections( - xyxy=xyxy, mask=masks, class_id=np.array(class_names) - ) + annotation = Detections(xyxy=xyxy, mask=np.array(masks), class_id=class_id) else: - annotation = Detections(xyxy=xyxy, class_id=np.array(class_names)) + annotation = Detections(xyxy=xyxy, class_id=class_id) images[image_path.name] = image annotations[image_path.name] = annotation - classes += class_names - - classes = list(set(classes)) - - for annotation in annotations.values(): - class_id = [classes.index(class_name) for class_name in annotation.class_id] - annotation.class_id = np.array(class_id) return classes, images, annotations From 1febd715e2df897b8b3cdcbc84d4029228f15596 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 22:53:24 +0200 Subject: [PATCH 31/54] remove test_detections_to_pascal_voc. conversion to XML is tested, and approx of masks should be in another test suite --- test/dataset/formats/test_pascal_voc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index f5a953a2..0d316d95 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -71,7 +71,3 @@ def test_object_to_pascal_voc( def test_load_pascal_voc_annotations(expected_result, exception: Exception): ... - - -def test_detections_to_pascal_voc(expected_result, exception: Exception): - ... From 16424325417f160269c5854c7142262e89001a8b Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 23:08:50 +0200 Subject: [PATCH 32/54] add test_parse_polygon_points --- supervision/dataset/formats/pascal_voc.py | 17 +++++++++++------ test/dataset/formats/test_pascal_voc.py | 23 +++++++++++++++++++++-- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index ad78246b..1cc0bfbf 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -181,12 +181,7 @@ def load_pascal_voc_annotations( with_masks = force_masks if force_masks else with_masks for polygon in obj.findall("polygon"): - polygon_points = [] - coords = polygon.findall(".//*") - for i in range(0, len(coords), 2): - x = int(coords[i].text) - y = int(coords[i + 1].text) - polygon_points.append([x, y]) + polygon_points = parse_polygon_points(polygon) mask_from_polygon = polygon_to_mask( polygon=np.array(polygon_points), @@ -211,6 +206,16 @@ def load_pascal_voc_annotations( return classes, images, annotations +def parse_polygon_points(polygon: Element): + polygon_points = [] + coords = polygon.findall(".//*") + for i in range(0, len(coords), 2): + x = int(coords[i].text) + y = int(coords[i + 1].text) + polygon_points.append([x, y]) + return polygon_points + + def load_pascal_voc_annotations_v1( annotation_path: str, ) -> Tuple[str, Detections, List[str]]: diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index 0d316d95..5d1dae69 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -10,6 +10,7 @@ from supervision.dataset.formats.pascal_voc import ( detections_to_pascal_voc, load_pascal_voc_annotations, object_to_pascal_voc, + parse_polygon_points, ) from supervision.detection.core import Detections @@ -69,5 +70,23 @@ def test_object_to_pascal_voc( assert are_xml_elements_equal(result, expected_result) -def test_load_pascal_voc_annotations(expected_result, exception: Exception): - ... +@pytest.mark.parametrize( + "polygon_element, expected_result, exception", + [ + ( + ET.fromstring( + """001001010010""" + ), + [[0, 0], [10, 0], [10, 10], [0, 10]], + DoesNotRaise(), + ) + ], +) +def test_parse_polygon_points( + polygon_element, + expected_result: List[list], + exception, +): + with exception: + result = parse_polygon_points(polygon_element) + assert result == expected_result From e4ef57ec2d6a5c455ff7cc50869f0254c5ca56e8 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 23:27:33 +0200 Subject: [PATCH 33/54] add test_detections_from_xml_obj --- supervision/dataset/formats/pascal_voc.py | 87 +++++++++++++---------- test/dataset/formats/test_pascal_voc.py | 23 ++++++ 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 1cc0bfbf..c325e80a 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -161,44 +161,10 @@ def load_pascal_voc_annotations( tree = parse(annotation_path) root = tree.getroot() - xyxy = [] - class_names = [] - masks = [] - with_masks = False - for obj in root.findall("object"): - class_name = obj.find("name").text - class_names.append(class_name) - - bbox = obj.find("bndbox") - x1 = int(bbox.find("xmin").text) - y1 = int(bbox.find("ymin").text) - x2 = int(bbox.find("xmax").text) - y2 = int(bbox.find("ymax").text) - - xyxy.append([x1, y1, x2, y2]) - - with_masks = obj.find("polygon") is not None - with_masks = force_masks if force_masks else with_masks - - for polygon in obj.findall("polygon"): - polygon_points = parse_polygon_points(polygon) - - mask_from_polygon = polygon_to_mask( - polygon=np.array(polygon_points), - resolution_wh=(image.shape[0], image.shape[1]), - ) - masks.append(mask_from_polygon) - - xyxy = np.array(xyxy) if len(xyxy) > 0 else np.empty((0, 4)) - for k in set(class_names): - if k not in classes: - classes.append(k) - class_id = np.array([classes.index(class_name) for class_name in class_names]) - - if with_masks: - annotation = Detections(xyxy=xyxy, mask=np.array(masks), class_id=class_id) - else: - annotation = Detections(xyxy=xyxy, class_id=class_id) + resolution_wh = (image.shape[0], image.shape[1]) + annotation, classes = detections_from_xml_obj( + root, classes, resolution_wh, force_masks + ) images[image_path.name] = image annotations[image_path.name] = annotation @@ -206,6 +172,51 @@ def load_pascal_voc_annotations( return classes, images, annotations +def detections_from_xml_obj(root, classes, resolution_wh, force_masks=False): + xyxy = [] + class_names = [] + masks = [] + with_masks = False + extended_classes = classes[:] + for obj in root.findall("object"): + class_name = obj.find("name").text + class_names.append(class_name) + + bbox = obj.find("bndbox") + x1 = int(bbox.find("xmin").text) + y1 = int(bbox.find("ymin").text) + x2 = int(bbox.find("xmax").text) + y2 = int(bbox.find("ymax").text) + + xyxy.append([x1, y1, x2, y2]) + + with_masks = obj.find("polygon") is not None + with_masks = force_masks if force_masks else with_masks + + for polygon in obj.findall("polygon"): + polygon_points = parse_polygon_points(polygon) + + mask_from_polygon = polygon_to_mask( + polygon=np.array(polygon_points), + resolution_wh=resolution_wh, + ) + masks.append(mask_from_polygon) + + xyxy = np.array(xyxy) if len(xyxy) > 0 else np.empty((0, 4)) + for k in set(class_names): + if k not in extended_classes: + extended_classes.append(k) + class_id = np.array( + [extended_classes.index(class_name) for class_name in class_names] + ) + + if with_masks: + annotation = Detections(xyxy=xyxy, mask=np.array(masks), class_id=class_id) + else: + annotation = Detections(xyxy=xyxy, class_id=class_id) + return annotation, extended_classes + + def parse_polygon_points(polygon: Element): polygon_points = [] coords = polygon.findall(".//*") diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index 5d1dae69..fa900f1c 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -7,6 +7,7 @@ import numpy as np import pytest from supervision.dataset.formats.pascal_voc import ( + detections_from_xml_obj, detections_to_pascal_voc, load_pascal_voc_annotations, object_to_pascal_voc, @@ -90,3 +91,25 @@ def test_parse_polygon_points( with exception: result = parse_polygon_points(polygon_element) assert result == expected_result + + +@pytest.mark.parametrize( + "xml_string, classes, resolution_wh, force_masks, expected_result, exception", + [ + ( + """test.jpg100100test001010""", + ["test"], + (100, 100), + False, + mock_detections(np.array([[0, 0, 10, 10]]), None, [0]), + DoesNotRaise(), + ) + ], +) +def test_detections_from_xml_obj( + xml_string, classes, resolution_wh, force_masks, expected_result, exception +): + with exception: + root = ET.fromstring(xml_string) + result, _ = detections_from_xml_obj(root, classes, resolution_wh, force_masks) + assert result == expected_result From f8005b268b86c661350feb818a87ea6768fb7cfd Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 23:39:42 +0200 Subject: [PATCH 34/54] add docstrings --- supervision/dataset/formats/pascal_voc.py | 34 +++++++++++++++++++++-- test/dataset/formats/test_pascal_voc.py | 2 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index c325e80a..53834d34 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -172,7 +172,36 @@ def load_pascal_voc_annotations( return classes, images, annotations -def detections_from_xml_obj(root, classes, resolution_wh, force_masks=False): +def detections_from_xml_obj( + root: Element, classes: List[str], resolution_wh, force_masks: bool = False +) -> Tuple[Detections, List[str]]: + """ + Converts an XML object in Pascal VOC format to a Detections object. + Expected XML format: + + ... + + dog + + 48 + 240 + 195 + 371 + + + 48 + 240 + 195 + 240 + 195 + 371 + 48 + 371 + + + + + """ xyxy = [] class_names = [] masks = [] @@ -217,7 +246,8 @@ def detections_from_xml_obj(root, classes, resolution_wh, force_masks=False): return annotation, extended_classes -def parse_polygon_points(polygon: Element): +def parse_polygon_points(polygon: Element) -> List[List[int]]: + # Parses polygon points in format: ............... polygon_points = [] coords = polygon.findall(".//*") for i in range(0, len(coords), 2): diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index fa900f1c..d1f25bd7 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -97,7 +97,7 @@ def test_parse_polygon_points( "xml_string, classes, resolution_wh, force_masks, expected_result, exception", [ ( - """test.jpg100100test001010""", + """test001010""", ["test"], (100, 100), False, From e8f616ba2fe4fef84aa5fdd2a487210e6be97e0e Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 23:41:02 +0200 Subject: [PATCH 35/54] cleanup imports --- test/dataset/formats/test_pascal_voc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index d1f25bd7..b12f2964 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -1,19 +1,16 @@ import xml.etree.ElementTree as ET from contextlib import ExitStack as DoesNotRaise from test.utils import mock_detections -from typing import List, Optional, Tuple +from typing import List, Optional import numpy as np import pytest from supervision.dataset.formats.pascal_voc import ( detections_from_xml_obj, - detections_to_pascal_voc, - load_pascal_voc_annotations, object_to_pascal_voc, parse_polygon_points, ) -from supervision.detection.core import Detections def are_xml_elements_equal(elem1, elem2): From 12b3def032142b0a466e3847a19f60ac5ec4224a Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Sat, 29 Jul 2023 23:42:59 +0200 Subject: [PATCH 36/54] upd docstring --- supervision/dataset/formats/pascal_voc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 53834d34..e915159c 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -201,6 +201,8 @@ def detections_from_xml_obj( + Returns: + Tuple[Detections, List[str]]: A tuple containing a Detections object and an updated list of class names, extended with the class names from the XML object. """ xyxy = [] class_names = [] From 95c8576aaf74a52f15afd769a5d08031d39b06bb Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 13:09:35 +0200 Subject: [PATCH 37/54] fix mask shape (N, W, H) -> (N, H, W) --- supervision/dataset/formats/pascal_voc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index e915159c..93e90927 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -161,7 +161,7 @@ def load_pascal_voc_annotations( tree = parse(annotation_path) root = tree.getroot() - resolution_wh = (image.shape[0], image.shape[1]) + resolution_wh = (image.shape[1], image.shape[0]) annotation, classes = detections_from_xml_obj( root, classes, resolution_wh, force_masks ) From 77c2afe55917526b201263de6012759a9382e43f Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 13:13:32 +0200 Subject: [PATCH 38/54] fix mask dtype --- supervision/detection/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index 63206ddc..cdca3ff7 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -17,8 +17,9 @@ def polygon_to_mask(polygon: np.ndarray, resolution_wh: Tuple[int, int]) -> np.n np.ndarray: The generated 2D mask, where the polygon is marked with `1`'s and the rest is filled with `0`'s. """ width, height = resolution_wh - mask = np.zeros((height, width), dtype=np.uint8) + mask = np.zeros((height, width)) cv2.fillPoly(mask, [polygon], color=1) + mask = mask.astype(bool) return mask From 9462a3058d3bf57cc428afc28f454ce645cf0d9e Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 13:15:40 +0200 Subject: [PATCH 39/54] cast masks to bool only when creating a Detection obj --- supervision/dataset/formats/pascal_voc.py | 3 +-- supervision/detection/utils.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 93e90927..18f130c7 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -242,14 +242,13 @@ def detections_from_xml_obj( ) if with_masks: - annotation = Detections(xyxy=xyxy, mask=np.array(masks), class_id=class_id) + annotation = Detections(xyxy=xyxy, mask=np.array(masks).astype(bool), class_id=class_id) else: annotation = Detections(xyxy=xyxy, class_id=class_id) return annotation, extended_classes def parse_polygon_points(polygon: Element) -> List[List[int]]: - # Parses polygon points in format: ............... polygon_points = [] coords = polygon.findall(".//*") for i in range(0, len(coords), 2): diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py index cdca3ff7..e30e6a05 100644 --- a/supervision/detection/utils.py +++ b/supervision/detection/utils.py @@ -19,7 +19,6 @@ def polygon_to_mask(polygon: np.ndarray, resolution_wh: Tuple[int, int]) -> np.n width, height = resolution_wh mask = np.zeros((height, width)) cv2.fillPoly(mask, [polygon], color=1) - mask = mask.astype(bool) return mask From c0c292ef8270196eac2752893c38047fc8de78f7 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 15:19:39 +0200 Subject: [PATCH 40/54] lint --- supervision/dataset/formats/pascal_voc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 18f130c7..17223d31 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -242,7 +242,9 @@ def detections_from_xml_obj( ) if with_masks: - annotation = Detections(xyxy=xyxy, mask=np.array(masks).astype(bool), class_id=class_id) + annotation = Detections( + xyxy=xyxy, mask=np.array(masks).astype(bool), class_id=class_id + ) else: annotation = Detections(xyxy=xyxy, class_id=class_id) return annotation, extended_classes From 36d3e94a769870ffbd416c2bbb8b736ae3e384b3 Mon Sep 17 00:00:00 2001 From: Hardik Dava <39372750+hardikdava@users.noreply.github.com> Date: Mon, 31 Jul 2023 18:51:02 +0200 Subject: [PATCH 41/54] removing unwanted code --- supervision/dataset/core.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/supervision/dataset/core.py b/supervision/dataset/core.py index 7167171a..ce7e0edb 100644 --- a/supervision/dataset/core.py +++ b/supervision/dataset/core.py @@ -46,14 +46,6 @@ class BaseDataset(ABC): ) -> Tuple[BaseDataset, BaseDataset]: pass - def add_instance( - self, filename: str, image: np.ndarray, detections: Detections - ) -> None: - pass - - def add_class_names(self, class_names: List[str]) -> None: - pass - @dataclass class DetectionDataset(BaseDataset): From e2d1fe7695a98a1e7fb69da96cd9326b41a20fba Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 19:42:53 +0200 Subject: [PATCH 42/54] remove artifacts --- test/dataset/formats/test_pascal_voc.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index b12f2964..8ae38801 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -61,10 +61,6 @@ def test_object_to_pascal_voc( ): with exception: result = object_to_pascal_voc(xyxy=xyxy, name=name, polygon=polygon) - with open("/tmp/test.xml", "w") as f: - f.write(ET.tostring(result).decode()) - with open("/tmp/exptest.xml", "w") as f: - f.write(ET.tostring(expected_result).decode()) assert are_xml_elements_equal(result, expected_result) From 26289e37439186ad18c39aedcff525d7cdb15340 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 19:43:32 +0200 Subject: [PATCH 43/54] drop load_pascal_voc_annotations_v1 --- supervision/dataset/formats/pascal_voc.py | 37 ----------------------- 1 file changed, 37 deletions(-) diff --git a/supervision/dataset/formats/pascal_voc.py b/supervision/dataset/formats/pascal_voc.py index 17223d31..4efb2917 100644 --- a/supervision/dataset/formats/pascal_voc.py +++ b/supervision/dataset/formats/pascal_voc.py @@ -258,40 +258,3 @@ def parse_polygon_points(polygon: Element) -> List[List[int]]: y = int(coords[i + 1].text) polygon_points.append([x, y]) return polygon_points - - -def load_pascal_voc_annotations_v1( - annotation_path: str, -) -> Tuple[str, Detections, List[str]]: - """ - Loads PASCAL VOC XML annotations and returns the image name, a Detections instance, and a list of class names. - - Args: - annotation_path (str): The path to the PASCAL VOC XML annotations file. - - Returns: - Tuple[str, Detections, List[str]]: A tuple containing the image name, a Detections instance, and a list of class names of objects in the detections. - """ - tree = parse(annotation_path) - root = tree.getroot() - - image_name = root.find("filename").text - - xyxy = [] - class_names = [] - for obj in root.findall("object"): - class_name = obj.find("name").text - class_names.append(class_name) - - bbox = obj.find("bndbox") - x1 = int(bbox.find("xmin").text) - y1 = int(bbox.find("ymin").text) - x2 = int(bbox.find("xmax").text) - y2 = int(bbox.find("ymax").text) - - xyxy.append([x1, y1, x2, y2]) - - xyxy = np.array(xyxy) - detections = Detections(xyxy=xyxy) - - return image_name, detections, class_names From 6fdabf0a964e5eda445256070c025a7797de5269 Mon Sep 17 00:00:00 2001 From: kirilllzaitsev Date: Mon, 31 Jul 2023 20:10:20 +0200 Subject: [PATCH 44/54] extend tests for test_detections_from_xml_obj --- test/dataset/formats/test_pascal_voc.py | 43 +++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/test/dataset/formats/test_pascal_voc.py b/test/dataset/formats/test_pascal_voc.py index 8ae38801..fa3f1124 100644 --- a/test/dataset/formats/test_pascal_voc.py +++ b/test/dataset/formats/test_pascal_voc.py @@ -86,17 +86,56 @@ def test_parse_polygon_points( assert result == expected_result +ONE_CLASS_N_BBOX = """test001010test10102020""" + + +ONE_CLASS_ONE_BBOX = """test001010""" + + +N_CLASS_N_BBOX = """test001010test20303040test210102020""" + +NO_DETECTIONS = """""" + + @pytest.mark.parametrize( "xml_string, classes, resolution_wh, force_masks, expected_result, exception", [ ( - """test001010""", + ONE_CLASS_ONE_BBOX, ["test"], (100, 100), False, mock_detections(np.array([[0, 0, 10, 10]]), None, [0]), DoesNotRaise(), - ) + ), + ( + ONE_CLASS_N_BBOX, + ["test"], + (100, 100), + False, + mock_detections(np.array([[0, 0, 10, 10], [10, 10, 20, 20]]), None, [0, 0]), + DoesNotRaise(), + ), + ( + N_CLASS_N_BBOX, + ["test", "test2"], + (100, 100), + False, + mock_detections( + np.array([[0, 0, 10, 10], [20, 30, 30, 40], [10, 10, 20, 20]]), + None, + [0, 0, 1], + ), + DoesNotRaise(), + ), + ( + NO_DETECTIONS, + [], + (100, 100), + False, + mock_detections(np.empty((0, 4)), None, []), + DoesNotRaise(), + ), ], ) def test_detections_from_xml_obj( From fa44884ea74cf85cefefd59a1d24438cd63fa9b7 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 31 Jul 2023 23:16:20 +0200 Subject: [PATCH 45/54] =?UTF-8?q?=F0=9F=92=AC=20update=20YOLOv8=20deprecat?= =?UTF-8?q?ed=20message?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/detection/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 033a41e5..4d7f8e5c 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -172,7 +172,7 @@ class Detections: @classmethod @deprecated( - "Please use sv.Detections.from_ultralytics() API for future usage. This method is deprecated and removed in future release" + "This method is deprecated and removed in 0.15.0 release. Use sv.Detections.from_ultralytics() instead." ) def from_yolov8(cls, yolov8_results) -> Detections: """ From b40e2f228ac07d20c4b74bc1214b36e05d54c80a Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Wed, 2 Aug 2023 14:40:16 +0200 Subject: [PATCH 46/54] =?UTF-8?q?=F0=9F=A7=B9=20small=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/dataset/formats/yolo.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/supervision/dataset/formats/yolo.py b/supervision/dataset/formats/yolo.py index da070a0f..0d41f045 100644 --- a/supervision/dataset/formats/yolo.py +++ b/supervision/dataset/formats/yolo.py @@ -126,9 +126,6 @@ def load_yolo_annotations( image_paths = list_files_with_extensions( directory=images_directory_path, extensions=["jpg", "jpeg", "png"] ) - annotation_paths = list_files_with_extensions( - directory=annotations_directory_path, extensions=["txt"] - ) classes = _extract_class_names(file_path=data_yaml_path) images = {} From 1ba1b67b4788455dbdacd8d74cb1377ef604a35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=2E=20Garc=C3=ADa-Oca=C3=B1a?= Date: Wed, 2 Aug 2023 18:51:49 +0200 Subject: [PATCH 47/54] Update core.py (n, W, H) -> (n, H, W) Fix mask array shape in docstrings (n, W, H) -> (n, H, W) #254 --- supervision/detection/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 410b216a..825d00c7 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -27,7 +27,7 @@ def _validate_mask(mask: Any, n: int) -> None: isinstance(mask, np.ndarray) and len(mask.shape) == 3 and mask.shape[0] == n ) if not is_valid: - raise ValueError("mask must be 3d np.ndarray with (n, W, H) shape") + raise ValueError("mask must be 3d np.ndarray with (n, H, W) shape") def _validate_class_id(class_id: Any, n: int) -> None: @@ -60,7 +60,7 @@ class Detections: Data class containing information about the detections in a video frame. Attributes: xyxy (np.ndarray): An array of shape `(n, 4)` containing the bounding boxes coordinates in format `[x1, y1, x2, y2]` - mask: (Optional[np.ndarray]): An array of shape `(n, W, H)` containing the segmentation masks. + mask: (Optional[np.ndarray]): An array of shape `(n, H, W)` containing the segmentation masks. confidence (Optional[np.ndarray]): An array of shape `(n,)` containing the confidence scores of the detections. class_id (Optional[np.ndarray]): An array of shape `(n,)` containing the class ids of the detections. tracker_id (Optional[np.ndarray]): An array of shape `(n,)` containing the tracker ids of the detections. From c6f2bf27c103eed3812614f9574c09c21d3d65c1 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Sun, 6 Aug 2023 11:12:42 +0200 Subject: [PATCH 48/54] =?UTF-8?q?=F0=9F=96=A4=20make=20black=20happy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/metrics/detection.py | 68 ++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index bb5e6f0e..1739f084 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -524,9 +524,7 @@ class MeanAveragePrecision: prediction_tensors.append( MeanAveragePrecision.detections_to_tensor(prediction) ) - target_tensors.append( - MeanAveragePrecision.targets_to_tensor(target) - ) + target_tensors.append(MeanAveragePrecision.targets_to_tensor(target)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, @@ -534,9 +532,9 @@ class MeanAveragePrecision: @classmethod def benchmark( - cls, - dataset: DetectionDataset, - callback: Callable[[np.ndarray], Detections], + cls, + dataset: DetectionDataset, + callback: Callable[[np.ndarray], Detections], ) -> MeanAveragePrecision: """ Get map from dataset and callback function. @@ -672,26 +670,35 @@ class MeanAveragePrecision: if len(stats) and stats[0].any(): average_precisions = cls.average_precisions_per_class(*stats) - ap50, ap75, average_precisions = average_precisions[:, 0], average_precisions[:, 5], average_precisions.mean(1) + ap50, ap75, average_precisions = ( + average_precisions[:, 0], + average_precisions[:, 5], + average_precisions.mean(1), + ) map50, map75, map = ap50.mean(), ap75.mean(), average_precisions.mean() - return cls(map=map, map50=map50, map75=map75, average_precisions=average_precisions) + return cls( + map=map, map50=map50, map75=map75, average_precisions=average_precisions + ) @staticmethod - def detections_to_tensor( - detections: Detections - ) -> np.ndarray: + def detections_to_tensor(detections: Detections) -> np.ndarray: if detections.class_id is None: raise ValueError( "MeanAveragePrecision can only be calculated for Detections with class_id" ) - return np.concatenate([detections.xyxy, np.expand_dims(detections.class_id, 1), np.expand_dims(detections.confidence, 1)], 1) + return np.concatenate( + [ + detections.xyxy, + np.expand_dims(detections.class_id, 1), + np.expand_dims(detections.confidence, 1), + ], + 1, + ) @staticmethod - def targets_to_tensor( - detections: Detections) -> np.ndarray: - + def targets_to_tensor(detections: Detections) -> np.ndarray: if detections.class_id is None: raise ValueError( "MeanAveragePrecision can only be calculated for Detections with class_id" @@ -711,7 +718,9 @@ class MeanAveragePrecision: x = np.where((iou >= iou_levels[i]) & correct_class) if x[0].shape[0]: - _X1 = np.concatenate([np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1) + _X1 = np.concatenate( + [np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1 + ) _x2 = iou[x[0], x[1]][:, None] matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] if x[0].shape[0] > 1: @@ -741,8 +750,14 @@ class MeanAveragePrecision: return ap @staticmethod - def average_precisions_per_class(matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16): - """ Compute the average precision, given the recall and precision curves. + def average_precisions_per_class( + matches: np.ndarray, + prediction_confidence: np.ndarray, + prediction_class_ids: np.ndarray, + true_batch_class_ids: np.ndarray, + EPS=1e-16, + ): + """Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. # Arguments matches: True positives (nparray, nx1 or nx10). @@ -755,7 +770,9 @@ class MeanAveragePrecision: prediction_class_ids = prediction_class_ids[sorted_confidences] # Find unique classes - unique_classes, class_counts = np.unique(true_batch_class_ids, return_counts=True) + unique_classes, class_counts = np.unique( + true_batch_class_ids, return_counts=True + ) num_classes = unique_classes.shape[0] # number of classes, number of detections average_precisions = np.zeros((num_classes, matches.shape[1])) @@ -773,7 +790,11 @@ class MeanAveragePrecision: precision = tp_pool / (tp_pool + fp_pool) for j in range(matches.shape[1]): - average_precisions[ci, j] = MeanAveragePrecision.compute_average_precision(recall[:, j], precision[:, j]) + average_precisions[ + ci, j + ] = MeanAveragePrecision.compute_average_precision( + recall[:, j], precision[:, j] + ) return average_precisions @@ -805,4 +826,9 @@ class MeanAveragePrecision: ) def to_dict(self): - return {'map': self.map, 'map50': self.map50, 'map75': self.map75, 'average_precisions': self.average_precisions} \ No newline at end of file + return { + "map": self.map, + "map50": self.map50, + "map75": self.map75, + "average_precisions": self.average_precisions, + } From fce6e660d9409722bc3e4aa9a7546a54ba930edf Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Sun, 6 Aug 2023 11:52:48 +0200 Subject: [PATCH 49/54] Small fixes and improvements. --- supervision/metrics/detection.py | 37 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 1739f084..94b196de 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -102,7 +102,7 @@ class ConfusionMatrix: def detections_to_tensor( detections: Detections, with_confidence: bool = False ) -> np.ndarray: - if detections == Detections.empty(): + if len(detections) == 0: if with_confidence: return np.zeros((0, 6)) else: @@ -325,7 +325,7 @@ class ConfusionMatrix: iou_threshold: float = 0.5, ) -> ConfusionMatrix: """ - Create confusion matrix from dataset and callback function. + Calculate confusion matrix from dataset and callback function. Args: dataset (DetectionDataset): Object detection dataset used for evaluation. @@ -466,19 +466,19 @@ class ConfusionMatrix: @dataclass(frozen=True) class MeanAveragePrecision: - map: float - map50: float - map75: float - average_precisions: np.ndarray """ Mean Average Precision for object detection tasks. Attributes: - map (float): map value. - map50 (float): map value at iou threshold=0.5. - map75 (float): map value at iou threshold=0.75 - average_precisions (np.ndarray): values for every classes + map (float): mAP value. + map50 (float): mAP value at IoU `threshold = 0.5`. + map75 (float): mAP value at IoU `threshold = 0.75`. + average_precisions (np.ndarray): values for every classes. """ + map: float + map50: float + map75: float + average_precisions: np.ndarray @classmethod def from_detections( @@ -487,7 +487,7 @@ class MeanAveragePrecision: targets: List[Detections], ) -> MeanAveragePrecision: """ - Calculate MeanAveragePrecision based on predicted and ground-truth detections. + Calculate mean average precision based on predicted and ground-truth detections. Args: targets (List[Detections]): Detections objects from ground-truth. @@ -509,12 +509,12 @@ class MeanAveragePrecision: ... sv.Detections(...) ... ] - >>> mean_average_precison = sv.MeanAveragePrecision.from_detections( + >>> mean_average_precision = sv.MeanAveragePrecision.from_detections( ... predictions=predictions, ... targets=target, ... ) - >>> mean_average_precison.matrix + >>> mean_average_precison.map 0.2899 ``` """ @@ -537,7 +537,7 @@ class MeanAveragePrecision: callback: Callable[[np.ndarray], Detections], ) -> MeanAveragePrecision: """ - Get map from dataset and callback function. + Calculate mean average precision from dataset and callback function. Args: dataset (DetectionDataset): Object detection dataset used for evaluation. @@ -669,7 +669,7 @@ class MeanAveragePrecision: stats = [np.concatenate(x, 0) for x in zip(*stats)] if len(stats) and stats[0].any(): - average_precisions = cls.average_precisions_per_class(*stats) + average_precisions = cls._average_precisions_per_class(*stats) ap50, ap75, average_precisions = ( average_precisions[:, 0], average_precisions[:, 5], @@ -750,16 +750,17 @@ class MeanAveragePrecision: return ap @staticmethod - def average_precisions_per_class( + def _average_precisions_per_class( matches: np.ndarray, prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16, ): - """Compute the average precision, given the recall and precision curves. + """ + Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. - # Arguments + Arguments matches: True positives (nparray, nx1 or nx10). prediction_confidence: Objectness value from 0-1 (nparray). prediction_class_ids: Predicted object classes (nparray). From 52cbe684d4a15466ff5d3d5aa5fa77945e183db7 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Sun, 6 Aug 2023 11:57:20 +0200 Subject: [PATCH 50/54] =?UTF-8?q?=F0=9F=96=A4=20MAke=20black=20happy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/metrics/detection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 94b196de..ba7c6c3f 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -475,6 +475,7 @@ class MeanAveragePrecision: map75 (float): mAP value at IoU `threshold = 0.75`. average_precisions (np.ndarray): values for every classes. """ + map: float map50: float map75: float From 0fcc17ee11156fd823b055f024e730c71a759334 Mon Sep 17 00:00:00 2001 From: Hardik Dava Date: Sun, 6 Aug 2023 12:23:00 +0200 Subject: [PATCH 51/54] Docstrings updated, return types fixed. --- supervision/metrics/detection.py | 104 +++++++++---------------------- 1 file changed, 30 insertions(+), 74 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index ba7c6c3f..a00f354b 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -523,9 +523,11 @@ class MeanAveragePrecision: target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - MeanAveragePrecision.detections_to_tensor(prediction) + ConfusionMatrix.detections_to_tensor(prediction, with_confidence=True) + ) + target_tensors.append( + ConfusionMatrix.detections_to_tensor(target, with_confidence=False) ) - target_tensors.append(MeanAveragePrecision.targets_to_tensor(target)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, @@ -631,7 +633,7 @@ class MeanAveragePrecision: 0.2899 ``` """ - cls._validate_input_tensors(predictions, targets) + ConfusionMatrix._validate_input_tensors(predictions, targets) map, map50, map75 = 0, 0, 0 class_index = 4 @@ -682,34 +684,19 @@ class MeanAveragePrecision: map=map, map50=map50, map75=map75, average_precisions=average_precisions ) - @staticmethod - def detections_to_tensor(detections: Detections) -> np.ndarray: - if detections.class_id is None: - raise ValueError( - "MeanAveragePrecision can only be calculated for Detections with class_id" - ) - - return np.concatenate( - [ - detections.xyxy, - np.expand_dims(detections.class_id, 1), - np.expand_dims(detections.confidence, 1), - ], - 1, - ) - - @staticmethod - def targets_to_tensor(detections: Detections) -> np.ndarray: - if detections.class_id is None: - raise ValueError( - "MeanAveragePrecision can only be calculated for Detections with class_id" - ) - return np.hstack([detections.xyxy, np.expand_dims(detections.class_id, 1)]) - @staticmethod def _match_detection_batch( predictions: np.ndarray, targets: np.ndarray, iou_levels: np.ndarray ) -> np.ndarray: + """ + Args: + predictions (np.ndarray): batch prediction + targets (np.ndarray): batch target labels + iou_levels (np.ndarray): iou levels array contains different iou levels + + Returns: + (np.ndarray): matched prediction with target lebels result + """ correct = np.zeros((predictions.shape[0], iou_levels.shape[0])).astype(bool) iou = box_iou_batch(targets[:, :4], predictions[:, :4]) @@ -733,13 +720,15 @@ class MeanAveragePrecision: return correct @staticmethod - def compute_average_precision(recall, precision): + def compute_average_precision( + recall: np.ndarray, precision: np.ndarray + ) -> np.ndarray: """Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves - # Arguments - recall: The recall curve (list) - precision: The precision curve (list) - # Returns - Average precision, precision curve, recall curve + Args: + recall (np.ndarray): The recall curve + precision (np.ndarray): The precision curve + Returns: + (np.ndarray) Average precision, precision curve, recall curve """ mrec = np.concatenate(([0.0], recall, [1.0])) mpre = np.concatenate(([1.0], precision, [0.0])) @@ -757,15 +746,17 @@ class MeanAveragePrecision: prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, EPS=1e-16, - ): + ) -> np.ndarray: """ Compute the average precision, given the recall and precision curves. Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. - Arguments - matches: True positives (nparray, nx1 or nx10). - prediction_confidence: Objectness value from 0-1 (nparray). - prediction_class_ids: Predicted object classes (nparray). - true_batch_class_ids: True object classes (nparray). + Args: + matches (np.ndarray): True positives (nparray, nx1 or nx10). + prediction_confidence (np.ndarray): Objectness value from 0-1 (nparray). + prediction_class_ids (np.ndarray): Predicted object classes (nparray). + true_batch_class_ids (np.ndarray): True object classes (nparray). + Returns: + (np.ndarray): Average precision for different iou level array """ sorted_confidences = np.argsort(-prediction_confidence) matches = matches[sorted_confidences] @@ -799,38 +790,3 @@ class MeanAveragePrecision: ) return average_precisions - - @classmethod - def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] - ): - """ - Checks for shape consistency of input tensors. - """ - if len(predictions) != len(targets): - raise ValueError( - f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." - ) - if len(predictions) > 0: - if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray - ): - raise ValueError( - f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." - ) - if predictions[0].shape[1] != 6: - raise ValueError( - f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." - ) - if targets[0].shape[1] != 5: - raise ValueError( - f"Targets must have shape (N, 5). Got {targets[0].shape} instead." - ) - - def to_dict(self): - return { - "map": self.map, - "map50": self.map50, - "map75": self.map75, - "average_precisions": self.average_precisions, - } From e0ca4ca5e755d923afead2e146d64cdbb3250855 Mon Sep 17 00:00:00 2001 From: Hardik Dava Date: Sun, 6 Aug 2023 12:30:13 +0200 Subject: [PATCH 52/54] added docstring and minor reformatting --- supervision/metrics/detection.py | 128 ++++++++++++++++--------------- 1 file changed, 65 insertions(+), 63 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index a00f354b..0b29bb07 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -12,6 +12,65 @@ from supervision.detection.core import Detections from supervision.detection.utils import box_iou_batch +def detections_to_tensor( + detections: Detections, with_confidence: bool = False +) -> np.ndarray: + """ + Convert Supervision Detections to numpy tensors for further computation + Args: + detections (sv.Detections): Detections/Targets in the format of sv.Detections + with_confidence (bool): Whether to include confidence in the tensor + Returns: + (np.ndarray): Detections as numpy tensors as in (xyxy, class_id, confidence) order + """ + if len(detections) == 0: + if with_confidence: + return np.zeros((0, 6)) + else: + return np.zeros((0, 5)) + + if detections.class_id is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with class_id" + ) + + arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] + + if with_confidence: + if detections.confidence is None: + raise ValueError( + "ConfusionMatrix can only be calculated for Detections with confidence" + ) + arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) + + return np.concatenate(arrays_to_concat, axis=1) + + +def _validate_input_tensors(predictions: List[np.ndarray], targets: List[np.ndarray]): + """ + Checks for shape consistency of input tensors. + """ + if len(predictions) != len(targets): + raise ValueError( + f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." + ) + if len(predictions) > 0: + if not isinstance(predictions[0], np.ndarray) or not isinstance( + targets[0], np.ndarray + ): + raise ValueError( + f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." + ) + if predictions[0].shape[1] != 6: + raise ValueError( + f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." + ) + if targets[0].shape[1] != 5: + raise ValueError( + f"Targets must have shape (N, 5). Got {targets[0].shape} instead." + ) + + @dataclass class ConfusionMatrix: """ @@ -85,11 +144,9 @@ class ConfusionMatrix: target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - ConfusionMatrix.detections_to_tensor(prediction, with_confidence=True) - ) - target_tensors.append( - ConfusionMatrix.detections_to_tensor(target, with_confidence=False) + detections_to_tensor(prediction, with_confidence=True) ) + target_tensors.append(detections_to_tensor(target, with_confidence=False)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, @@ -98,32 +155,6 @@ class ConfusionMatrix: iou_threshold=iou_threshold, ) - @staticmethod - def detections_to_tensor( - detections: Detections, with_confidence: bool = False - ) -> np.ndarray: - if len(detections) == 0: - if with_confidence: - return np.zeros((0, 6)) - else: - return np.zeros((0, 5)) - - if detections.class_id is None: - raise ValueError( - "ConfusionMatrix can only be calculated for Detections with class_id" - ) - - arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)] - - if with_confidence: - if detections.confidence is None: - raise ValueError( - "ConfusionMatrix can only be calculated for Detections with confidence" - ) - arrays_to_concat.append(np.expand_dims(detections.confidence, 1)) - - return np.concatenate(arrays_to_concat, axis=1) - @classmethod def from_tensors( cls, @@ -190,7 +221,7 @@ class ConfusionMatrix: ]) ``` """ - cls._validate_input_tensors(predictions, targets) + _validate_input_tensors(predictions, targets) num_classes = len(classes) matrix = np.zeros((num_classes + 1, num_classes + 1)) @@ -209,33 +240,6 @@ class ConfusionMatrix: iou_threshold=iou_threshold, ) - @classmethod - def _validate_input_tensors( - cls, predictions: List[np.ndarray], targets: List[np.ndarray] - ): - """ - Checks for shape consistency of input tensors. - """ - if len(predictions) != len(targets): - raise ValueError( - f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal." - ) - if len(predictions) > 0: - if not isinstance(predictions[0], np.ndarray) or not isinstance( - targets[0], np.ndarray - ): - raise ValueError( - f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead." - ) - if predictions[0].shape[1] != 6: - raise ValueError( - f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead." - ) - if targets[0].shape[1] != 5: - raise ValueError( - f"Targets must have shape (N, 5). Got {targets[0].shape} instead." - ) - @staticmethod def evaluate_detection_batch( predictions: np.ndarray, @@ -523,11 +527,9 @@ class MeanAveragePrecision: target_tensors = [] for prediction, target in zip(predictions, targets): prediction_tensors.append( - ConfusionMatrix.detections_to_tensor(prediction, with_confidence=True) - ) - target_tensors.append( - ConfusionMatrix.detections_to_tensor(target, with_confidence=False) + detections_to_tensor(prediction, with_confidence=True) ) + target_tensors.append(detections_to_tensor(target, with_confidence=False)) return cls.from_tensors( predictions=prediction_tensors, targets=target_tensors, @@ -633,7 +635,7 @@ class MeanAveragePrecision: 0.2899 ``` """ - ConfusionMatrix._validate_input_tensors(predictions, targets) + _validate_input_tensors(predictions, targets) map, map50, map75 = 0, 0, 0 class_index = 4 From 3a31ea2986f9de12fee0ffe2ae1a930af04f1fca Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Sun, 6 Aug 2023 13:24:51 +0200 Subject: [PATCH 53/54] More small changes --- supervision/metrics/detection.py | 16 +++++----------- test/metrics/test_detection.py | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 0b29bb07..153da248 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -23,12 +23,6 @@ def detections_to_tensor( Returns: (np.ndarray): Detections as numpy tensors as in (xyxy, class_id, confidence) order """ - if len(detections) == 0: - if with_confidence: - return np.zeros((0, 6)) - else: - return np.zeros((0, 5)) - if detections.class_id is None: raise ValueError( "ConfusionMatrix can only be calculated for Detections with class_id" @@ -46,7 +40,7 @@ def detections_to_tensor( return np.concatenate(arrays_to_concat, axis=1) -def _validate_input_tensors(predictions: List[np.ndarray], targets: List[np.ndarray]): +def validate_input_tensors(predictions: List[np.ndarray], targets: List[np.ndarray]): """ Checks for shape consistency of input tensors. """ @@ -221,7 +215,7 @@ class ConfusionMatrix: ]) ``` """ - _validate_input_tensors(predictions, targets) + validate_input_tensors(predictions, targets) num_classes = len(classes) matrix = np.zeros((num_classes + 1, num_classes + 1)) @@ -521,7 +515,7 @@ class MeanAveragePrecision: >>> mean_average_precison.map 0.2899 - ``` + ``` """ prediction_tensors = [] target_tensors = [] @@ -635,7 +629,7 @@ class MeanAveragePrecision: 0.2899 ``` """ - _validate_input_tensors(predictions, targets) + validate_input_tensors(predictions, targets) map, map50, map75 = 0, 0, 0 class_index = 4 @@ -730,7 +724,7 @@ class MeanAveragePrecision: recall (np.ndarray): The recall curve precision (np.ndarray): The precision curve Returns: - (np.ndarray) Average precision, precision curve, recall curve + (np.ndarray): Average precision, precision curve, recall curve """ mrec = np.concatenate(([0.0], recall, [1.0])) mpre = np.concatenate(([1.0], precision, [0.0])) diff --git a/test/metrics/test_detection.py b/test/metrics/test_detection.py index 0ecab3f0..1d4a5740 100644 --- a/test/metrics/test_detection.py +++ b/test/metrics/test_detection.py @@ -5,7 +5,7 @@ import numpy as np import pytest from supervision.detection.core import Detections -from supervision.metrics.detection import ConfusionMatrix +from supervision.metrics.detection import ConfusionMatrix, detections_to_tensor from test.utils import mock_detections CLASSES = np.arange(80) @@ -167,7 +167,7 @@ def test_detections_to_tensor( exception: Exception ): with exception: - result = ConfusionMatrix.detections_to_tensor( + result = detections_to_tensor( detections=detections, with_confidence=with_confidence ) From 7f4662cf3d44bc7e2d9049ee4970d8a83ed25c52 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 7 Aug 2023 09:58:46 +0200 Subject: [PATCH 54/54] Final changes before merge --- supervision/metrics/detection.py | 52 +++++++++++++++++--------------- test/metrics/test_detection.py | 44 +++++++++++++++++++++++++-- test/utils.py | 4 +++ 3 files changed, 73 insertions(+), 27 deletions(-) diff --git a/supervision/metrics/detection.py b/supervision/metrics/detection.py index 153da248..d126f82e 100644 --- a/supervision/metrics/detection.py +++ b/supervision/metrics/detection.py @@ -471,13 +471,13 @@ class MeanAveragePrecision: map (float): mAP value. map50 (float): mAP value at IoU `threshold = 0.5`. map75 (float): mAP value at IoU `threshold = 0.75`. - average_precisions (np.ndarray): values for every classes. + per_class_ap (np.ndarray): values for every classes. """ map: float map50: float map75: float - average_precisions: np.ndarray + per_class_ap: np.ndarray @classmethod def from_detections( @@ -644,14 +644,14 @@ class MeanAveragePrecision: true_batch.shape[0], detection_batch.shape[0], ) - correct = np.zeros((npr, num_ious), dtype=bool) # init + correct = np.zeros((npr, num_ious), dtype=bool) if npr == 0: if nl: stats.append((correct, *np.zeros((2, 0)), true_batch[:, 4])) continue if nl: - correct = cls._match_detection_batch( + correct = MeanAveragePrecision._match_detection_batch( predictions=detection_batch, targets=true_batch, iou_levels=iou_levels, @@ -676,9 +676,7 @@ class MeanAveragePrecision: ) map50, map75, map = ap50.mean(), ap75.mean(), average_precisions.mean() - return cls( - map=map, map50=map50, map75=map75, average_precisions=average_precisions - ) + return cls(map=map, map50=map50, map75=map75, per_class_ap=average_precisions) @staticmethod def _match_detection_batch( @@ -706,7 +704,7 @@ class MeanAveragePrecision: [np.expand_dims(x[0], 1), np.expand_dims(x[1], 1)], axis=1 ) _x2 = iou[x[0], x[1]][:, None] - matches = np.concatenate([_X1, _x2], axis=1) # [label, detect, iou] + matches = np.concatenate([_X1, _x2], axis=1) if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] @@ -716,24 +714,28 @@ class MeanAveragePrecision: return correct @staticmethod - def compute_average_precision( - recall: np.ndarray, precision: np.ndarray - ) -> np.ndarray: - """Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves - Args: - recall (np.ndarray): The recall curve - precision (np.ndarray): The precision curve - Returns: - (np.ndarray): Average precision, precision curve, recall curve + def compute_average_precision(recall: np.ndarray, precision: np.ndarray) -> float: """ - mrec = np.concatenate(([0.0], recall, [1.0])) - mpre = np.concatenate(([1.0], precision, [0.0])) + Compute the average precision using 101-point interpolation (COCO), given the recall and precision curves. - mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + Args: + recall (np.ndarray): The recall curve. + precision (np.ndarray): The precision curve. - x = np.linspace(0, 1, 101) - ap = np.trapz(np.interp(x, mrec, mpre), x) - return ap + Returns: + float: Average precision. + """ + extended_recall = np.concatenate(([0.0], recall, [1.0])) + extended_precision = np.concatenate(([1.0], precision, [0.0])) + max_accumulated_precision = np.flip( + np.maximum.accumulate(np.flip(extended_precision)) + ) + interpolated_recall_levels = np.linspace(0, 1, 101) + interpolated_precision = np.interp( + interpolated_recall_levels, extended_recall, max_accumulated_precision + ) + average_precision = np.trapz(interpolated_precision, interpolated_recall_levels) + return average_precision @staticmethod def _average_precisions_per_class( @@ -741,7 +743,7 @@ class MeanAveragePrecision: prediction_confidence: np.ndarray, prediction_class_ids: np.ndarray, true_batch_class_ids: np.ndarray, - EPS=1e-16, + eps: float = 1e-16, ) -> np.ndarray: """ Compute the average precision, given the recall and precision curves. @@ -775,7 +777,7 @@ class MeanAveragePrecision: fp_pool = (1 - matches[valid]).cumsum(0) tp_pool = matches[valid].cumsum(0) - recall = tp_pool / (num_targets + EPS) + recall = tp_pool / (num_targets + eps) precision = tp_pool / (tp_pool + fp_pool) for j in range(matches.shape[1]): diff --git a/test/metrics/test_detection.py b/test/metrics/test_detection.py index 1d4a5740..e0a5f2ce 100644 --- a/test/metrics/test_detection.py +++ b/test/metrics/test_detection.py @@ -5,8 +5,8 @@ import numpy as np import pytest from supervision.detection.core import Detections -from supervision.metrics.detection import ConfusionMatrix, detections_to_tensor -from test.utils import mock_detections +from supervision.metrics.detection import ConfusionMatrix, detections_to_tensor, MeanAveragePrecision +from test.utils import mock_detections, assert_almost_equal CLASSES = np.arange(80) NUM_CLASSES = len(CLASSES) @@ -400,3 +400,43 @@ def test_drop_extra_matches( result = ConfusionMatrix._drop_extra_matches(matches) assert np.array_equal(result, expected_result) + + +@pytest.mark.parametrize( + 'recall, precision, expected_result, exception', + [ + ( + np.array([1.0]), + np.array([1.0]), + 1.0, + DoesNotRaise() + ), # perfect recall and precision + ( + np.array([0.0]), + np.array([0.0]), + 0.0, + DoesNotRaise() + ), # no recall and precision + ( + np.array([0.0, 0.2, 0.2, 0.8, 0.8, 1.0]), + np.array([0.7, 0.8, 0.4, 0.5, 0.1, 0.2]), + 0.5, + DoesNotRaise() + ), + ( + np.array([0.0, 0.5, 0.5, 1.0]), + np.array([0.75, 0.75, 0.75, 0.75]), + 0.75, + DoesNotRaise() + ) + ] +) +def test_compute_average_precision( + recall: np.ndarray, + precision: np.ndarray, + expected_result: float, + exception: Exception +) -> None: + with exception: + result = MeanAveragePrecision.compute_average_precision(recall=recall, precision=precision) + assert_almost_equal(result, expected_result, tolerance=0.01) diff --git a/test/utils.py b/test/utils.py index 20c294e5..a1d6907e 100644 --- a/test/utils.py +++ b/test/utils.py @@ -21,3 +21,7 @@ def mock_detections( if tracker_id is None else np.array(tracker_id, dtype=int), ) + + +def assert_almost_equal(actual, expected, tolerance=1e-5): + assert abs(actual - expected) < tolerance, f"Expected {expected}, but got {actual}."