From fbd96d10c4c6358ddba4bf2bd4d34f498ebfde2a Mon Sep 17 00:00:00 2001 From: LinasKo Date: Fri, 18 Oct 2024 15:15:40 +0300 Subject: [PATCH] Add new and missing docstrings, examples to metrics, new Common section --- docs/metrics/common_values.md | 20 +++++++ docs/metrics/precision.md | 2 +- docs/metrics/recall.md | 2 +- mkdocs.yml | 1 + supervision/metrics/core.py | 26 +++++---- supervision/metrics/f1_score.py | 55 +++++++++++++++++- supervision/metrics/mean_average_precision.py | 44 ++++++++------ supervision/metrics/precision.py | 58 ++++++++++++++++++- supervision/metrics/recall.py | 58 ++++++++++++++++++- 9 files changed, 231 insertions(+), 35 deletions(-) create mode 100644 docs/metrics/common_values.md diff --git a/docs/metrics/common_values.md b/docs/metrics/common_values.md new file mode 100644 index 00000000..b7600f3f --- /dev/null +++ b/docs/metrics/common_values.md @@ -0,0 +1,20 @@ +--- +comments: true +status: new +--- + +# Common Values + +This page contains supplementary values, types and enums that metrics use. + +
+

MetricTarget

+
+ +:::supervision.metrics.core.MetricTarget + +
+

AveragingMethod

+
+ +:::supervision.metrics.core.AveragingMethod diff --git a/docs/metrics/precision.md b/docs/metrics/precision.md index ca318f8f..c704452e 100644 --- a/docs/metrics/precision.md +++ b/docs/metrics/precision.md @@ -3,7 +3,7 @@ comments: true status: new --- -# F1 Score +# Precision

Precision

diff --git a/docs/metrics/recall.md b/docs/metrics/recall.md index 5baa4d3e..78dde833 100644 --- a/docs/metrics/recall.md +++ b/docs/metrics/recall.md @@ -3,7 +3,7 @@ comments: true status: new --- -# F1 Score +# Recall

Recall

diff --git a/mkdocs.yml b/mkdocs.yml index a3c9c1ca..b30dbcfc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -69,6 +69,7 @@ nav: - Precision: metrics/precision.md - Recall: metrics/recall.md - F1 Score: metrics/f1_score.md + - Common Values: metrics/common_values.md - Legacy Metrics: detection/metrics.md - Utils: - Video: utils/video.md diff --git a/supervision/metrics/core.py b/supervision/metrics/core.py index d1818441..def5999a 100644 --- a/supervision/metrics/core.py +++ b/supervision/metrics/core.py @@ -37,9 +37,10 @@ class MetricTarget(Enum): """ Specifies what type of detection is used to compute the metric. - * BOXES: xyxy bounding boxes - * MASKS: Binary masks - * ORIENTED_BOUNDING_BOXES: Oriented bounding boxes (OBB) + Attributes: + BOXES: xyxy bounding boxes + MASKS: Binary masks + ORIENTED_BOUNDING_BOXES: Oriented bounding boxes (OBB) """ BOXES = "boxes" @@ -54,15 +55,16 @@ class AveragingMethod(Enum): Suppose, before returning the final result, a metric is computed for each class. How do you combine those to get the final number? - * MACRO: Calculate the metric for each class and average the results. The simplest - averaging method, but it does not take class imbalance into account. - * MICRO: Calculate the metric globally by counting the total true positives, false - positives, and false negatives. Micro averaging is useful when you want to give - more importance to classes with more samples. It's also more appropriate if you - have an imbalance in the number of instances per class. - * WEIGHTED: Calculate the metric for each class and average the results, weighted by - the number of true instances of each class. Use weighted averaging if you want - to take class imbalance into account. + Attributes: + MACRO: Calculate the metric for each class and average the results. The simplest + averaging method, but it does not take class imbalance into account. + MICRO: Calculate the metric globally by counting the total true positives, false + positives, and false negatives. Micro averaging is useful when you want to + give more importance to classes with more samples. It's also more + appropriate if you have an imbalance in the number of instances per class. + WEIGHTED: Calculate the metric for each class and average the results, weighted + by the number of true instances of each class. Use weighted averaging if + you want to take class imbalance into account. """ MACRO = "macro" diff --git a/supervision/metrics/f1_score.py b/supervision/metrics/f1_score.py index 2ca5bca5..ba4fcd59 100644 --- a/supervision/metrics/f1_score.py +++ b/supervision/metrics/f1_score.py @@ -23,11 +23,45 @@ if TYPE_CHECKING: class F1Score(Metric): + """ + F1 Score is a metric used to evaluate object detection models. It is the harmonic + mean of precision and recall, calculated at different IoU thresholds. + + In simple terms, F1 Score is a measure of a model's balance between precision and + recall (accuracy and completeness), calculated as: + + `F1 = 2 * (precision * recall) / (precision + recall)` + + Example: + ```python + import supervision as sv + from supervision.metrics import F1Score + + predictions = sv.Detections(...) + targets = sv.Detections(...) + + f1_metric = F1Score() + f1_result = f1_metric.update(predictions, targets).compute() + + print(f1_result) + print(f1_result.f1_50) + print(f1_result.small_objects.f1_50) + ``` + """ + def __init__( self, metric_target: MetricTarget = MetricTarget.BOXES, averaging_method: AveragingMethod = AveragingMethod.WEIGHTED, ): + """ + Initialize the F1Score metric. + + Args: + metric_target (MetricTarget): The type of detection data to use. + averaging_method (AveragingMethod): The averaging method used to compute the + F1 scores. Determines how the F1 scores are aggregated across classes. + """ self._metric_target = metric_target if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: raise NotImplementedError( @@ -40,6 +74,9 @@ class F1Score(Metric): self._targets_list: List[Detections] = [] def reset(self) -> None: + """ + Reset the metric to its initial state, clearing all stored data. + """ self._predictions_list = [] self._targets_list = [] @@ -48,6 +85,16 @@ class F1Score(Metric): predictions: Union[Detections, List[Detections]], targets: Union[Detections, List[Detections]], ) -> F1Score: + """ + Add new predictions and targets to the metric, but do not compute the result. + + Args: + predictions (Union[Detections, List[Detections]]): The predicted detections. + targets (Union[Detections, List[Detections]]): The target detections. + + Returns: + (F1Score): The updated metric instance. + """ if not isinstance(predictions, list): predictions = [predictions] if not isinstance(targets, list): @@ -65,6 +112,13 @@ class F1Score(Metric): return self def compute(self) -> F1ScoreResult: + """ + Calculate the F1 score metric based on the stored predictions and ground-truth + data, at different IoU thresholds. + + Returns: + (F1ScoreResult): The F1 score metric result. + """ result = self._compute(self._predictions_list, self._targets_list) small_predictions, small_targets = self._filter_predictions_and_targets_by_size( @@ -373,7 +427,6 @@ class F1ScoreResult: The results of the F1 score metric calculation. Defaults to `0` if no detections or targets were provided. - Provides a custom `__str__` method for pretty printing. Attributes: metric_target (MetricTarget): the type of data used for the metric - diff --git a/supervision/metrics/mean_average_precision.py b/supervision/metrics/mean_average_precision.py index dbd60b2e..8cec50c8 100644 --- a/supervision/metrics/mean_average_precision.py +++ b/supervision/metrics/mean_average_precision.py @@ -23,6 +23,27 @@ if TYPE_CHECKING: class MeanAveragePrecision(Metric): + """ + Mean Average Precision (mAP) is a metric used to evaluate object detection models. + It is the average of the precision-recall curves at different IoU thresholds. + + Example: + ```python + import supervision as sv + from supervision.metrics import MeanAveragePrecision + + predictions = sv.Detections(...) + targets = sv.Detections(...) + + map_metric = MeanAveragePrecision() + map_result = map_metric.update(predictions, targets).compute() + + print(map_result) + print(map_result.map50_95) + map_result.plot() + ``` + """ + def __init__( self, metric_target: MetricTarget = MetricTarget.BOXES, @@ -47,6 +68,9 @@ class MeanAveragePrecision(Metric): self._targets_list: List[Detections] = [] def reset(self) -> None: + """ + Reset the metric to its initial state, clearing all stored data. + """ self._predictions_list = [] self._targets_list = [] @@ -95,26 +119,10 @@ class MeanAveragePrecision(Metric): ) -> MeanAveragePrecisionResult: """ Calculate Mean Average Precision based on predicted and ground-truth - detections at different thresholds. + detections at different thresholds. Returns: - (MeanAveragePrecisionResult): New instance of MeanAveragePrecision. - - Example: - ```python - import supervision as sv - from supervision.metrics import MeanAveragePrecision - - predictions = sv.Detections(...) - targets = sv.Detections(...) - - map_metric = MeanAveragePrecision() - map_result = map_metric.update(predictions, targets).compute() - - print(map_result) - print(map_result.map50_95) - map_result.plot() - ``` + (MeanAveragePrecisionResult): The Mean Average Precision result. """ result = self._compute(self._predictions_list, self._targets_list) diff --git a/supervision/metrics/precision.py b/supervision/metrics/precision.py index ba441831..d915e1f4 100644 --- a/supervision/metrics/precision.py +++ b/supervision/metrics/precision.py @@ -23,11 +23,48 @@ if TYPE_CHECKING: class Precision(Metric): + """ + Precision is a metric used to evaluate object detection models. It is the ratio of + true positive detections to the total number of predicted detections. We calculate + it at different IoU thresholds. + + In simple terms, Precision is a measure of a model's accuracy, calculated as: + + `Precision = TP / (TP + FP)` + + Here, `TP` is the number of true positives (correct detections), and `FP` is the + number of false positive detections (detected, but incorrectly). + + Example: + ```python + import supervision as sv + from supervision.metrics import Precision + + predictions = sv.Detections(...) + targets = sv.Detections(...) + + precision_metric = Precision() + precision_result = precision_metric.update(predictions, targets).compute() + + print(precision_result) + print(precision_result.precision_at_50) + print(precision_result.small_objects.precision_at_50) + ``` + """ + def __init__( self, metric_target: MetricTarget = MetricTarget.BOXES, averaging_method: AveragingMethod = AveragingMethod.WEIGHTED, ): + """ + Initialize the Precision metric. + + Args: + metric_target (MetricTarget): The type of detection data to use. + averaging_method (AveragingMethod): The averaging method used to compute the + precision. Determines how the precision is aggregated across classes. + """ self._metric_target = metric_target if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: raise NotImplementedError( @@ -40,6 +77,9 @@ class Precision(Metric): self._targets_list: List[Detections] = [] def reset(self) -> None: + """ + Reset the metric to its initial state, clearing all stored data. + """ self._predictions_list = [] self._targets_list = [] @@ -48,6 +88,16 @@ class Precision(Metric): predictions: Union[Detections, List[Detections]], targets: Union[Detections, List[Detections]], ) -> Precision: + """ + Add new predictions and targets to the metric, but do not compute the result. + + Args: + predictions (Union[Detections, List[Detections]]): The predicted detections. + targets (Union[Detections, List[Detections]]): The target detections. + + Returns: + (Precision): The updated metric instance. + """ if not isinstance(predictions, list): predictions = [predictions] if not isinstance(targets, list): @@ -65,6 +115,13 @@ class Precision(Metric): return self def compute(self) -> PrecisionResult: + """ + Calculate the precision metric based on the stored predictions and ground-truth + data, at different IoU thresholds. + + Returns: + (PrecisionResult): The precision metric result. + """ result = self._compute(self._predictions_list, self._targets_list) small_predictions, small_targets = self._filter_predictions_and_targets_by_size( @@ -373,7 +430,6 @@ class PrecisionResult: The results of the precision metric calculation. Defaults to `0` if no detections or targets were provided. - Provides a custom `__str__` method for pretty printing. Attributes: metric_target (MetricTarget): the type of data used for the metric - diff --git a/supervision/metrics/recall.py b/supervision/metrics/recall.py index 7c90859c..9eae24f8 100644 --- a/supervision/metrics/recall.py +++ b/supervision/metrics/recall.py @@ -23,11 +23,48 @@ if TYPE_CHECKING: class Recall(Metric): + """ + Recall is a metric used to evaluate object detection models. It is the ratio of + true positive detections to the total number of ground truth instances. We calculate + it at different IoU thresholds. + + In simple terms, Recall is a measure of a model's completeness, calculated as: + + `Recall = TP / (TP + FN)` + + Here, `TP` is the number of true positives (correct detections), and `FN` is the + number of false negatives (missed detections). + + Example: + ```python + import supervision as sv + from supervision.metrics import Recall + + predictions = sv.Detections(...) + targets = sv.Detections(...) + + recall_metric = Recall() + recall_result = recall_metric.update(predictions, targets).compute() + + print(recall_result) + print(recall_result.recall_at_50) + print(recall_result.small_objects.recall_at_50) + ``` + """ + def __init__( self, metric_target: MetricTarget = MetricTarget.BOXES, averaging_method: AveragingMethod = AveragingMethod.WEIGHTED, ): + """ + Initialize the Recall metric. + + Args: + metric_target (MetricTarget): The type of detection data to use. + averaging_method (AveragingMethod): The averaging method used to compute the + recall. Determines how the recall is aggregated across classes. + """ self._metric_target = metric_target if self._metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES: raise NotImplementedError( @@ -40,6 +77,9 @@ class Recall(Metric): self._targets_list: List[Detections] = [] def reset(self) -> None: + """ + Reset the metric to its initial state, clearing all stored data. + """ self._predictions_list = [] self._targets_list = [] @@ -48,6 +88,16 @@ class Recall(Metric): predictions: Union[Detections, List[Detections]], targets: Union[Detections, List[Detections]], ) -> Recall: + """ + Add new predictions and targets to the metric, but do not compute the result. + + Args: + predictions (Union[Detections, List[Detections]]): The predicted detections. + targets (Union[Detections, List[Detections]]): The target detections. + + Returns: + (Recall): The updated metric instance. + """ if not isinstance(predictions, list): predictions = [predictions] if not isinstance(targets, list): @@ -65,6 +115,13 @@ class Recall(Metric): return self def compute(self) -> RecallResult: + """ + Calculate the precision metric based on the stored predictions and ground-truth + data, at different IoU thresholds. + + Returns: + (RecallResult): The precision metric result. + """ result = self._compute(self._predictions_list, self._targets_list) small_predictions, small_targets = self._filter_predictions_and_targets_by_size( @@ -371,7 +428,6 @@ class RecallResult: The results of the recall metric calculation. Defaults to `0` if no detections or targets were provided. - Provides a custom `__str__` method for pretty printing. Attributes: metric_target (MetricTarget): the type of data used for the metric -