xiaoxue_admin
/
supervision
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:hardikdava/supervision into main

This commit is contained in:
hd 2023-07-22 10:44:12 +02:00
parent fcde1cea9c 3d074fa213
commit e9e075acaf
20 changed files with 4094 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/docs.yml vendored
View File

@ -9,10 +9,10 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.x
- run: pip install mkdocs-material
- run: pip install "mkdocstrings[python]"
- run: mkdocs gh-deploy --force
- run: mkdocs gh-deploy --force

31
.github/workflows/publish.yml vendored
View File

@ -16,17 +16,32 @@ jobs:
with:
ref: ${{ github.head_ref }}
- name: 🐍 Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: 🦾 Install dependencies
run: |
python -m pip install --upgrade virtualenv
python -m pip install --upgrade pip
pip install ".[dev]"
- name: 🚀 Publish to PyPi
env:
PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
PYPI_TEST_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }}
virtualenv venv
source venv/bin/activate
python -m pip install --upgrade pip
python -m pip install --upgrade poetry
poetry install
- name: 🏗️ Build source and wheel distributions
run: |
make publish -e PYPI_USERNAME=$PYPI_USERNAME -e PYPI_PASSWORD=$PYPI_PASSWORD -e PYPI_TEST_PASSWORD=$PYPI_TEST_PASSWORD
python -m pip install --upgrade build twine
python -m build
twine check --strict dist/*
- name: 🚀 Publish to PyPi
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: ${{ secrets.PYPI_USERNAME }}
password: ${{ secrets.PYPI_PASSWORD }}
- name: 🚀 Publish to Test-PyPi
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
user: ${{ secrets.PYPI_USERNAME }}
password: ${{ secrets.PYPI_TEST_PASSWORD }}

4
.github/workflows/test.yml vendored
View File

@ -9,14 +9,14 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.7, 3.8, 3.9, '3.10']
python-version: ["3.8", "3.9", "3.10","3.11"]
steps:
- name: 🛎️ Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: 🐍 Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: 🦾 Install dependencies

2
Makefile
View File

@ -16,7 +16,7 @@ check_code_quality:
flake8 $(check_dirs) --count --max-line-length=88 --exit-zero --ignore=D --extend-ignore=E203,E501,W503 --statistics
publish:
python setup.py sdist bdist_wheel
poetry build
twine upload -r testpypi dist/* -u ${PYPI_USERNAME} -p ${PYPI_TEST_PASSWORD} --verbose
twine check dist/*
twine upload dist/* -u ${PYPI_USERNAME} -p ${PYPI_PASSWORD} --verbose

2
README.md
View File

@ -25,7 +25,7 @@
## 💻 install
Pip install the supervision package in a
[**3.11>=Python>=3.7**](https://www.python.org/) environment.
[**3.11>=Python>=3.8**](https://www.python.org/) environment.
```bash
pip install supervision

8
docs/metrics/detection.md Normal file
View File

@ -0,0 +1,8 @@
!!! warning
Evaluation API is still fluid and may change. If you use Evaluation API in your project until further notice, freeze the
`supervision` version in your `requirements.txt` or `setup.py`.
## ConfusionMatrix
:::supervision.metrics.detection.ConfusionMatrix

2
mkdocs.yml
View File

@ -39,6 +39,8 @@ nav:
- Polygon Zone: detection/tools/polygon_zone.md
- Dataset:
- Core: dataset/core.md
- Metrics:
- Detection Models: metrics/detection.md
- Draw:
- Utils: draw/utils.md
- Utils:

3121
poetry.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

66
pyproject.toml Normal file
View File

@ -0,0 +1,66 @@
[tool.poetry]
name = "supervision"
version = "0.12.0"
description = "A set of easy-to-use utils that will come in handy in any Computer Vision project"
authors = ["Piotr Skalski <piotr.skalski92@gmail.com>"]
maintainers = ["Piotr Skalski <piotr.skalski92@gmail.com>"]
readme = "README.md"
packages = [{include = "supervision"}]
homepage = "https://github.com/roboflow/supervision"
repository = "https://github.com/roboflow/supervision"
documentation = "https://github.com/roboflow/supervision/blob/main/README.md"
keywords = ["machine-learning", "deep-learning", "vision", "ML", "DL", "AI", "YOLOv5", "YOLOv8", "Roboflow"]
classifiers=[
'Intended Audience :: Developers',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: BSD License',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3 :: Only',
'Topic :: Software Development',
'Topic :: Scientific/Engineering',
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Scientific/Engineering :: Image Recognition",
'Typing :: Typed',
'Operating System :: Microsoft :: Windows',
'Operating System :: POSIX',
'Operating System :: Unix',
'Operating System :: MacOS'
]
[tool.poetry.dependencies]
python = ">=3.8,<3.12.0"
numpy = "^1.20.0"
matplotlib = "^3.7.1"
pyyaml = "^6.0"
pillow = "^8.4.0"
opencv-python = { version = "^4.8.0.74", optional = true }
opencv-python-headless = "^4.8.0.74"
[tool.poetry.extras]
desktop = ["opencv-python"]
[tool.poetry.group.dev.dependencies]
twine = "^4.0.2"
pytest = "^7.2.2"
wheel = "^0.40.0"
notebook = "^6.5.3"
mkdocs-material = "^9.1.4"
mkdocstrings = {extras = ["python"], version = "^0.20.0"}
build = "^0.10.0"
[tool.setuptools]
include-package-data = false
[tool.setuptools.packages.find]
exclude = ["docs*", "test*"]
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

71
setup.py
View File

@ -1,71 +0,0 @@
import setuptools
from setuptools import find_packages
import re
from pathlib import Path
FILE = Path(__file__).resolve()
PARENT = FILE.parent # root directory
README = (PARENT / "README.md").read_text(encoding="utf-8")
def get_version():
file = PARENT / 'supervision/__init__.py'
return re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', file.read_text(encoding="utf-8"), re.M)[1]
setuptools.setup(
name='supervision',
version=get_version(),
author='Piotr Skalski',
author_email='piotr.skalski92@gmail.com',
license='MIT',
description='A set of easy-to-use utils that will come in handy in any Computer Vision project',
long_description=README,
long_description_content_type='text/markdown',
url='https://github.com/roboflow/supervision',
install_requires=[
'numpy>=1.20.0',
'opencv-python',
'matplotlib',
'pyyaml'
],
packages=find_packages(exclude=("tests",)),
extras_require={
'dev': [
'flake8',
'black==22.3.0',
'isort',
'twine',
'pytest',
'wheel',
'notebook',
'mkdocs-material',
'mkdocstrings[python]'
],
},
classifiers=[
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'Intended Audience :: Education',
'Intended Audience :: Science/Research',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3 :: Only',
'Topic :: Software Development',
'Topic :: Scientific/Engineering',
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Scientific/Engineering :: Image Recognition",
'Typing :: Typed',
'Operating System :: Microsoft :: Windows',
'Operating System :: POSIX',
'Operating System :: Unix',
'Operating System :: MacOS'
],
keywords="machine-learning, deep-learning, vision, ML, DL, AI, YOLOv5, YOLOv8, SAM, Roboflow",
python_requires='>=3.7',
)

6
supervision/__init__.py
View File

@ -1,4 +1,7 @@
__version__ = "0.11.1"
import importlib.metadata as importlib_metadata
__version__ = importlib_metadata.version(__package__)
from supervision.classification.core import Classifications
from supervision.dataset.core import (
@ -23,6 +26,7 @@ from supervision.draw.color import Color, ColorPalette
from supervision.draw.utils import draw_filled_rectangle, draw_polygon, draw_text
from supervision.geometry.core import Point, Position, Rect
from supervision.geometry.utils import get_polygon_center
from supervision.metrics.detection import ConfusionMatrix
from supervision.utils.file import list_files_with_extensions
from supervision.utils.image import ImageSink, crop
from supervision.utils.notebook import plot_image, plot_images_grid

8
supervision/dataset/utils.py
View File

@ -83,9 +83,11 @@ def map_detections_class_id(
)
detections_copy = copy.deepcopy(detections)
detections_copy.class_id = np.vectorize(source_to_target_mapping.get)(
detections_copy.class_id
)
if len(detections) > 0:
detections_copy.class_id = np.vectorize(source_to_target_mapping.get)(
detections_copy.class_id
)
return detections_copy

29
supervision/detection/core.py
View File

@ -228,6 +228,35 @@ class Detections:
class_id=yolo_nas_results.prediction.labels.astype(int),
)
@classmethod
def from_mmdetection(cls, mmdet_results) -> Detections:
"""
Creates a Detections instance from a [mmdetection](https://github.com/open-mmlab/mmdetection) inference result.
Also supported for [mmyolo](https://github.com/open-mmlab/mmyolo)
Args:
mmdet_results (mmdet.structures.DetDataSample): The output Results instance from MMDetection
Returns:
Detections: A new Detections object.
Example:
```python
>>> import cv2
>>> import supervision as sv
>>> from mmdet.apis import DetInferencer
>>> inferencer = DetInferencer(model_name, checkpoint, device)
>>> mmdet_result = inferencer(SOURCE_IMAGE_PATH, out_dir='./output', return_datasample=True)["predictions"][0]
>>> detections = sv.Detections.from_mmdet(mmdet_result)
```
"""
return cls(
xyxy=mmdet_results.pred_instances.bboxes.cpu().numpy(),
confidence=mmdet_results.pred_instances.scores.cpu().numpy(),
class_id=mmdet_results.pred_instances.labels.cpu().numpy().astype(int),
)
@classmethod
def from_transformers(cls, transformers_results: dict) -> Detections:
"""

0
supervision/metrics/__init__.py Normal file
View File

445
supervision/metrics/detection.py Normal file
View File

@ -0,0 +1,445 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable, List, Optional, Tuple
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from supervision.dataset.core import DetectionDataset
from supervision.detection.core import Detections
from supervision.detection.utils import box_iou_batch
@dataclass
class ConfusionMatrix:
"""
Confusion matrix for object detection tasks.
Attributes:
matrix (np.ndarray): An 2D `np.ndarray` of shape `(len(classes) + 1, len(classes) + 1)` containing the number of `TP`, `FP`, `FN` and `TN` for each class.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded from the matrix.
iou_threshold (float): Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`.
"""
matrix: np.ndarray
classes: List[str]
conf_threshold: float
iou_threshold: float
@classmethod
def from_detections(
cls,
predictions: List[Detections],
targets: List[Detections],
classes: List[str],
conf_threshold: float = 0.3,
iou_threshold: float = 0.5,
) -> ConfusionMatrix:
"""
Calculate confusion matrix based on predicted and ground-truth detections.
Args:
targets (List[Detections]): Detections objects from ground-truth.
predictions (List[Detections]): Detections objects predicted by the model.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded.
iou_threshold (float): Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
Example:
```python
>>> import supervision as sv
>>> targets = [
... sv.Detections(...),
... sv.Detections(...)
... ]
>>> predictions = [
... sv.Detections(...),
... sv.Detections(...)
... ]
>>> confusion_matrix = sv.ConfusionMatrix.from_detections(
... predictions=predictions,
... targets=target,
... classes=['person', ...]
... )
>>> confusion_matrix.matrix
array([
[0., 0., 0., 0.],
[0., 1., 0., 1.],
[0., 1., 1., 0.],
[1., 1., 0., 0.]
])
```
"""
prediction_tensors = []
target_tensors = []
for prediction, target in zip(predictions, targets):
prediction_tensors.append(cls.convert_detections_to_tensor(prediction))
target_tensors.append(cls.convert_detections_to_tensor(target))
return cls.from_tensors(
predictions=prediction_tensors,
targets=target_tensors,
classes=classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
@classmethod
def convert_detections_to_tensor(cls, detections: Detections) -> np.ndarray:
arrays_to_concat = [detections.xyxy, np.expand_dims(detections.class_id, 1)]
if detections.confidence is not None:
arrays_to_concat.append(np.expand_dims(detections.confidence, 1))
return np.concatenate(
arrays_to_concat,
axis=1,
)
@classmethod
def from_tensors(
cls,
predictions: List[np.ndarray],
targets: List[np.ndarray],
classes: List[str],
conf_threshold: float = 0.3,
iou_threshold: float = 0.5,
) -> ConfusionMatrix:
"""
Calculate confusion matrix based on predicted and ground-truth detections.
Args:
predictions (List[np.ndarray]): Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format.
targets (List[np.ndarray]): Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format.
classes (List[str]): Model class names.
conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded.
iou_threshold (float): Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
Example:
```python
>>> import supervision as sv
>>> targets = (
... [
... array(
... [
... [0.0, 0.0, 3.0, 3.0, 1],
... [2.0, 2.0, 5.0, 5.0, 1],
... [6.0, 1.0, 8.0, 3.0, 2],
... ]
... ),
... array([1.0, 1.0, 2.0, 2.0, 2]),
... ]
... )
>>> predictions = [
... array(
... [
... [0.0, 0.0, 3.0, 3.0, 1, 0.9],
... [0.1, 0.1, 3.0, 3.0, 0, 0.9],
... [6.0, 1.0, 8.0, 3.0, 1, 0.8],
... [1.0, 6.0, 2.0, 7.0, 1, 0.8],
... ]
... ),
... array([[1.0, 1.0, 2.0, 2.0, 2, 0.8]])
... ]
>>> confusion_matrix = sv.ConfusionMatrix.from_tensors(
... predictions=predictions,
... targets=targets,
... classes=['person', ...]
... )
>>> confusion_matrix.matrix
array([
[0., 0., 0., 0.],
[0., 1., 0., 1.],
[0., 1., 1., 0.],
[1., 1., 0., 0.]
])
```
"""
cls._validate_input_tensors(predictions, targets)
num_classes = len(classes)
matrix = np.zeros((num_classes + 1, num_classes + 1))
for true_batch, detection_batch in zip(targets, predictions):
matrix += cls.evaluate_detection_batch(
predictions=detection_batch,
targets=true_batch,
num_classes=num_classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
return cls(
matrix=matrix,
classes=classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
@classmethod
def _validate_input_tensors(
cls, predictions: List[np.ndarray], targets: List[np.ndarray]
):
"""
Checks for shape consistency of input tensors.
"""
if len(predictions) != len(targets):
raise ValueError(
f"Number of predictions ({len(predictions)}) and targets ({len(targets)}) must be equal."
)
if len(predictions) > 0:
if not isinstance(predictions[0], np.ndarray) or not isinstance(
targets[0], np.ndarray
):
raise ValueError(
f"Predictions and targets must be lists of numpy arrays. Got {type(predictions[0])} and {type(targets[0])} instead."
)
if predictions[0].shape[1] != 6:
raise ValueError(
f"Predictions must have shape (N, 6). Got {predictions[0].shape} instead."
)
if targets[0].shape[1] != 5:
raise ValueError(
f"Targets must have shape (N, 5). Got {targets[0].shape} instead."
)
@staticmethod
def evaluate_detection_batch(
predictions: np.ndarray,
targets: np.ndarray,
num_classes: int,
conf_threshold: float,
iou_threshold: float,
) -> np.ndarray:
"""
Calculate confusion matrix for a batch of detections for a single image.
Args:
predictions (List[np.ndarray]): Each element of the list describes a single image and has `shape = (M, 6)` where `M` is the number of detected objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class, conf)` format.
targets (List[np.ndarray]): Each element of the list describes a single image and has `shape = (N, 5)` where `N` is the number of ground-truth objects. Each row is expected to be in `(x_min, y_min, x_max, y_max, class)` format.
num_classes (int): Number of classes.
conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded.
iou_threshold (float): Detection iou threshold between `0` and `1`. Detections with lower iou will be classified as `FP`.
Returns:
np.ndarray: Confusion matrix based on a single image.
"""
result_matrix = np.zeros((num_classes + 1, num_classes + 1))
conf_idx = 5
confidence = predictions[:, conf_idx]
detection_batch_filtered = predictions[confidence > conf_threshold]
class_id_idx = 4
true_classes = np.array(targets[:, class_id_idx], dtype=np.int16)
detection_classes = np.array(
detection_batch_filtered[:, class_id_idx], dtype=np.int16
)
true_boxes = targets[:, :class_id_idx]
detection_boxes = detection_batch_filtered[:, :class_id_idx]
iou_batch = box_iou_batch(
boxes_true=true_boxes, boxes_detection=detection_boxes
)
matched_idx = np.asarray(iou_batch > iou_threshold).nonzero()
if matched_idx[0].shape[0]:
matches = np.stack(
(matched_idx[0], matched_idx[1], iou_batch[matched_idx]), axis=1
)
matches = ConfusionMatrix._drop_extra_matches(matches=matches)
else:
matches = np.zeros((0, 3))
matched_true_idx, matched_detection_idx, _ = matches.transpose().astype(
np.int16
)
for i, true_class_value in enumerate(true_classes):
j = matched_true_idx == i
if matches.shape[0] > 0 and sum(j) == 1:
result_matrix[
true_class_value, detection_classes[matched_detection_idx[j]]
] += 1 # TP
else:
result_matrix[true_class_value, num_classes] += 1 # FN
for i, detection_class_value in enumerate(detection_classes):
if not any(matched_detection_idx == i):
result_matrix[num_classes, detection_class_value] += 1 # FP
return result_matrix
@staticmethod
def _drop_extra_matches(matches: np.ndarray) -> np.ndarray:
"""
Deduplicate matches. If there are multiple matches for the same true or predicted box,
only the one with the highest IoU is kept.
"""
if matches.shape[0] > 0:
matches = matches[matches[:, 2].argsort()[::-1]]
matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
matches = matches[matches[:, 2].argsort()[::-1]]
matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
return matches
@classmethod
def benchmark(
cls,
dataset: DetectionDataset,
callback: Callable[[np.ndarray], Detections],
conf_threshold: float = 0.3,
iou_threshold: float = 0.5,
) -> ConfusionMatrix:
"""
Create confusion matrix from dataset and callback function.
Args:
dataset (DetectionDataset): Object detection dataset used for evaluation.
callback (Callable[[np.ndarray], Detections]): Function that takes an image as input and returns Detections object.
conf_threshold (float): Detection confidence threshold between `0` and `1`. Detections with lower confidence will be excluded.
iou_threshold (float): Detection IoU threshold between `0` and `1`. Detections with lower IoU will be classified as `FP`.
Returns:
ConfusionMatrix: New instance of ConfusionMatrix.
Example:
```python
>>> import supervision as sv
>>> from ultralytics import YOLO
>>> dataset = sv.DetectionDataset.from_yolo(...)
>>> model = YOLO(...)
>>> def callback(image: np.ndarray) -> sv.Detections:
... result = model(image)[0]
... return sv.Detections.from_yolov8(result)
>>> confusion_matrix = sv.ConfusionMatrix.benchmark(
... dataset = dataset,
... callback = callback
... )
>>> confusion_matrix.matrix
array([
[0., 0., 0., 0.],
[0., 1., 0., 1.],
[0., 1., 1., 0.],
[1., 1., 0., 0.]
])
```
"""
predictions, targets = [], []
for img_name, img in dataset.images.items():
predictions_batch = callback(img)
predictions.append(predictions_batch)
targets_batch = dataset.annotations[img_name]
targets.append(targets_batch)
return cls.from_detections(
predictions=predictions,
targets=targets,
classes=dataset.classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
def plot(
self,
save_path: Optional[str] = None,
title: Optional[str] = None,
classes: Optional[List[str]] = None,
normalize: bool = False,
fig_size: Tuple[int, int] = (12, 10),
) -> matplotlib.figure.Figure:
"""
Create confusion matrix plot and save it at selected location.
Args:
save_path (Optional[str]): Path to save the plot. If not provided, plot will be displayed.
title (Optional[str]): Title of the plot.
classes (Optional[List[str]]): List of classes to be displayed on the plot. If not provided, all classes will be displayed.
normalize (bool): If True, normalize the confusion matrix.
fig_size (Tuple[int, int]): Size of the plot.
Returns:
matplotlib.figure.Figure: Confusion matrix plot.
"""
array = self.matrix.copy()
if normalize:
eps = 1e-8
array = array / (array.sum(0).reshape(1, -1) + eps)
array[array < 0.005] = np.nan
fig, ax = plt.subplots(figsize=fig_size, tight_layout=True, facecolor="white")
class_names = classes if classes is not None else self.classes
use_labels_for_ticks = class_names is not None and (0 < len(class_names) < 99)
if use_labels_for_ticks:
x_tick_labels = class_names + ["FN"]
y_tick_labels = class_names + ["FP"]
num_ticks = len(x_tick_labels)
else:
x_tick_labels = None
y_tick_labels = None
num_ticks = len(array)
im = ax.imshow(array, cmap="Blues")
cbar = ax.figure.colorbar(im, ax=ax)
cbar.mappable.set_clim(vmin=0, vmax=np.nanmax(array))
if x_tick_labels is None:
tick_interval = 2
else:
tick_interval = 1
ax.set_xticks(np.arange(0, num_ticks, tick_interval), labels=x_tick_labels)
ax.set_yticks(np.arange(0, num_ticks, tick_interval), labels=y_tick_labels)
plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="default")
labelsize = 10 if num_ticks < 50 else 8
ax.tick_params(axis="both", which="both", labelsize=labelsize)
if num_ticks < 30:
for i in range(array.shape[0]):
for j in range(array.shape[1]):
n_preds = array[i, j]
if not np.isnan(n_preds):
ax.text(
j,
i,
f"{n_preds:.2f}" if normalize else f"{n_preds:.0f}",
ha="center",
va="center",
color="black"
if n_preds < 0.5 * np.nanmax(array)
else "white",
)
if title:
ax.set_title(title, fontsize=20)
ax.set_xlabel("Predicted")
ax.set_ylabel("True")
ax.set_facecolor("white")
if save_path:
fig.savefig(
save_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True
)
return fig

1
test/dataset/test_core.py
View File

@ -172,5 +172,4 @@ def test_dataset_merge(
) -> None:
with exception:
result = DetectionDataset.merge(dataset_list=dataset_list)
print(result.images.keys())
assert result == expected_result

6
test/dataset/test_utils.py
View File

@ -204,6 +204,12 @@ def test_build_class_index_mapping(
mock_detections(xyxy=[[0, 0, 10, 10]], class_id=[1]),
DoesNotRaise()
), # single mapping
(
{0: 1, 1: 2},
Detections.empty(),
Detections.empty(),
DoesNotRaise()
), # empty detections
(
{0: 1, 1: 2},
mock_detections(xyxy=[[0, 0, 10, 10]], class_id=[0]),

0
test/metrics/__init__.py Normal file
View File

369
test/metrics/test_detection.py Normal file
View File

@ -0,0 +1,369 @@
from contextlib import ExitStack as DoesNotRaise
from typing import Optional, Union
import numpy as np
import pytest
from supervision.detection.core import Detections
from supervision.metrics.detection import ConfusionMatrix
CLASSES = np.arange(80)
NUM_CLASSES = len(CLASSES)
PREDICTIONS = np.array(
[
[2254, 906, 2447, 1353, 0.90538, 0],
[2049, 1133, 2226, 1371, 0.59002, 56],
[727, 1224, 838, 1601, 0.51119, 39],
[808, 1214, 910, 1564, 0.45287, 39],
[6, 52, 1131, 2133, 0.45057, 72],
[299, 1225, 512, 1663, 0.45029, 39],
[529, 874, 645, 945, 0.31101, 39],
[8, 47, 1935, 2135, 0.28192, 72],
[2265, 813, 2328, 901, 0.2714, 62],
],
dtype=np.float32,
)
TARGET_TENSORS = [
np.array(
[
[2254, 906, 2447, 1353, 0],
[2049, 1133, 2226, 1371, 56],
[727, 1224, 838, 1601, 39],
[808, 1214, 910, 1564, 39],
[6, 52, 1131, 2133, 72],
[299, 1225, 512, 1663, 39],
[529, 874, 645, 945, 39],
[8, 47, 1935, 2135, 72],
[2265, 813, 2328, 901, 62],
]
)
]
DETECTIONS = Detections(
xyxy=PREDICTIONS[:, :4],
confidence=PREDICTIONS[:, 4],
class_id=PREDICTIONS[:, 5].astype(int),
)
CERTAIN_DETECTIONS = Detections(
xyxy=PREDICTIONS[:, :4],
confidence=np.ones(len(PREDICTIONS)),
class_id=PREDICTIONS[:, 5].astype(int),
)
DETECTION_TENSORS = [
np.concatenate(
[
det.xyxy,
np.expand_dims(det.class_id, 1),
np.expand_dims(det.confidence, 1),
],
axis=1,
)
for det in [DETECTIONS]
]
CERTAIN_DETECTION_TENSORS = [
np.concatenate(
[
det.xyxy,
np.expand_dims(det.class_id, 1),
np.ones((len(det), 1)),
],
axis=1,
)
for det in [DETECTIONS]
]
IDEAL_MATCHES = np.stack(
[
np.arange(len(PREDICTIONS)),
np.arange(len(PREDICTIONS)),
np.ones(len(PREDICTIONS)),
],
axis=1,
)
def create_empty_conf_matrix(num_classes: int, do_add_dummy_class: bool = True):
if do_add_dummy_class:
num_classes += 1
return np.zeros((num_classes, num_classes))
def update_ideal_conf_matrix(conf_matrix: np.ndarray, class_ids: np.ndarray):
for class_id, count in zip(*np.unique(class_ids, return_counts=True)):
class_id = int(class_id)
conf_matrix[class_id, class_id] += count
return conf_matrix
def worsen_ideal_conf_matrix(
conf_matrix: np.ndarray, class_ids: Union[np.ndarray, list]
):
for class_id in class_ids:
class_id = int(class_id)
conf_matrix[class_id, class_id] -= 1
conf_matrix[class_id, 80] += 1
return conf_matrix
IDEAL_CONF_MATRIX = create_empty_conf_matrix(NUM_CLASSES)
IDEAL_CONF_MATRIX = update_ideal_conf_matrix(IDEAL_CONF_MATRIX, PREDICTIONS[:, 5])
GOOD_CONF_MATRIX = worsen_ideal_conf_matrix(IDEAL_CONF_MATRIX.copy(), [62, 72])
BAD_CONF_MATRIX = worsen_ideal_conf_matrix(
IDEAL_CONF_MATRIX.copy(), [62, 72, 72, 39, 39, 39, 39, 56]
)
@pytest.mark.parametrize(
"detections, exception",
[
(
DETECTIONS,
DoesNotRaise(),
)
],
)
def test_convert_detections_to_tensor(
detections,
exception: Exception,
):
with exception:
result = ConfusionMatrix.convert_detections_to_tensor(
detections=detections,
)
assert np.array_equal(result[:, :4], detections.xyxy)
assert np.array_equal(result[:, 4], detections.class_id)
assert np.array_equal(result[:, 5], detections.confidence)
@pytest.mark.parametrize(
"predictions, targets, classes, conf_threshold, iou_threshold, expected_result, exception",
[
(
DETECTION_TENSORS,
TARGET_TENSORS,
CLASSES,
0.2,
0.5,
IDEAL_CONF_MATRIX,
DoesNotRaise(),
),
(
[],
[],
CLASSES,
0.2,
0.5,
create_empty_conf_matrix(NUM_CLASSES),
DoesNotRaise(),
),
(
DETECTION_TENSORS,
TARGET_TENSORS,
CLASSES,
0.3,
0.5,
GOOD_CONF_MATRIX,
DoesNotRaise(),
),
(
DETECTION_TENSORS,
TARGET_TENSORS,
CLASSES,
0.6,
0.5,
BAD_CONF_MATRIX,
DoesNotRaise(),
),
(
[
np.array(
[
[0.0, 0.0, 3.0, 3.0, 0, 0.9], # correct detection of [0]
[
0.1,
0.1,
3.0,
3.0,
0,
0.9,
], # additional detection of [0] - FP
[
6.0,
1.0,
8.0,
3.0,
1,
0.8,
], # correct detection with incorrect class
[1.0, 6.0, 2.0, 7.0, 1, 0.8], # incorrect detection - FP
[
1.0,
2.0,
2.0,
4.0,
1,
0.8,
], # incorrect detection with low IoU - FP
]
)
],
[
np.array(
[
[0.0, 0.0, 3.0, 3.0, 0], # [0] detected
[2.0, 2.0, 5.0, 5.0, 1], # [1] undetected - FN
[
6.0,
1.0,
8.0,
3.0,
2,
], # [2] correct detection with incorrect class
]
)
],
CLASSES[:3],
0.6,
0.5,
np.array([[1, 0, 0, 0], [0, 0, 0, 1], [0, 1, 0, 0], [1, 2, 0, 0]]),
DoesNotRaise(),
),
(
[
np.array(
[
[0.0, 0.0, 3.0, 3.0, 0, 0.9], # correct detection of [0]
[
0.1,
0.1,
3.0,
3.0,
0,
0.9,
], # additional detection of [0] - FP
[
6.0,
1.0,
8.0,
3.0,
1,
0.8,
], # correct detection with incorrect class
[1.0, 6.0, 2.0, 7.0, 1, 0.8], # incorrect detection - FP
[
1.0,
2.0,
2.0,
4.0,
1,
0.8,
], # incorrect detection with low IoU - FP
]
)
],
[
np.array(
[
[0.0, 0.0, 3.0, 3.0, 0], # [0] detected
[2.0, 2.0, 5.0, 5.0, 1], # [1] undetected - FN
[
6.0,
1.0,
8.0,
3.0,
2,
], # [2] correct detection with incorrect class
]
)
],
CLASSES[:3],
0.6,
1.0,
np.array([[0, 0, 0, 1], [0, 0, 0, 1], [0, 0, 0, 1], [2, 3, 0, 0]]),
DoesNotRaise(),
),
],
)
def test_from_tensors(
predictions,
targets,
classes,
conf_threshold,
iou_threshold,
expected_result: Optional[np.ndarray],
exception: Exception,
):
with exception:
result = ConfusionMatrix.from_tensors(
predictions=predictions,
targets=targets,
classes=classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
assert result.matrix.diagonal().sum() == expected_result.diagonal().sum()
assert np.array_equal(result.matrix, expected_result)
@pytest.mark.parametrize(
"predictions, targets, num_classes, conf_threshold, iou_threshold, expected_result, exception",
[
(
DETECTION_TENSORS[0],
CERTAIN_DETECTION_TENSORS[0],
NUM_CLASSES,
0.2,
0.5,
IDEAL_CONF_MATRIX,
DoesNotRaise(),
)
],
)
def test_evaluate_detection_batch(
predictions,
targets,
num_classes,
conf_threshold,
iou_threshold,
expected_result: Optional[np.ndarray],
exception: Exception,
):
with exception:
result = ConfusionMatrix.evaluate_detection_batch(
predictions=predictions,
targets=targets,
num_classes=num_classes,
conf_threshold=conf_threshold,
iou_threshold=iou_threshold,
)
assert result.diagonal().sum() == result.sum()
assert np.array_equal(result, expected_result)
@pytest.mark.parametrize(
"matches, expected_result, exception",
[
(
IDEAL_MATCHES,
IDEAL_MATCHES,
DoesNotRaise(),
)
],
)
def test_drop_extra_matches(
matches,
expected_result: Optional[np.ndarray],
exception: Exception,
):
with exception:
result = ConfusionMatrix._drop_extra_matches(matches)
assert np.array_equal(result, expected_result)

12
test/utils.py
View File

@ -2,18 +2,22 @@ from typing import List
import numpy as np
from supervision import Detections
from supervision.detection.core import Detections
def mock_detections(
xyxy: List[List[float]],
confidence: List[float] = None,
class_id: List[int] = None,
tracker_id: List[int] = None
tracker_id: List[int] = None,
) -> Detections:
return Detections(
xyxy=np.array(xyxy, dtype=np.float32),
confidence=confidence if confidence is None else np.array(confidence, dtype=np.float32),
confidence=confidence
if confidence is None
else np.array(confidence, dtype=np.float32),
class_id=class_id if class_id is None else np.array(class_id, dtype=int),
tracker_id=tracker_id if tracker_id is None else np.array(tracker_id, dtype=int)
tracker_id=tracker_id
if tracker_id is None
else np.array(tracker_id, dtype=int),
)