diff --git a/supervision/detection/core.py b/supervision/detection/core.py index ea466b89..ffe5ed3f 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -40,6 +40,7 @@ from supervision.detection.utils.masks import calculate_masks_centroids from supervision.detection.vlm import ( LMM, VLM, + from_deepseek_vl_2, from_florence_2, from_google_gemini_2_0, from_google_gemini_2_5, @@ -830,6 +831,7 @@ class Detections: | Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` | | Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` | | Moondream | `MOONDREAM` | detection | `resolution_wh` | | + | DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` | Args: lmm (Union[LMM, str]): The type of LMM (Large Multimodal Model) to use. @@ -1117,6 +1119,47 @@ class Detections: # array([[1752.28, 818.82, 2165.72, 1229.14], # [1908.01, 1346.67, 2585.99, 2024.11]]) ``` + + !!! example "DeepSeek-VL2" + + + ??? tip "Prompt engineering" + + To get the best results from DeepSeek-VL2, use optimized prompts that leverage + its object detection and visual grounding capabilities effectively. + + **For general object detection, use the following user prompt:** + + ``` + \\n<|ref|>The giraffe at the front<|/ref|> + ``` + + **For visual grounding, use the following user prompt:** + + ``` + \\n<|grounding|>Detect the giraffes + ``` + + ```python + from PIL import Image + import supervision as sv + + deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>" + + detections = sv.Detections.from_vlm( + vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size + ) + + detections.xyxy + # array([[ 420, 293, 724, 982], + # [ 18, 33, 458, 1084]]) + + detections.class_id + # array([0, 1]) + + detections.data + # {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype='\\n<|ref|>The giraffe at the front<|/ref|> + ``` + + **For visual grounding, use the following user prompt:** + + ``` + \\n<|grounding|>Detect the giraffes + ``` + + ```python + from PIL import Image + import supervision as sv + + deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>" + + detections = sv.Detections.from_vlm( + vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size + ) + + detections.xyxy + # array([[ 420, 293, 724, 982], + # [ 18, 33, 458, 1084]]) + + detections.class_id + # array([0, 1]) + + detections.data + # {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype=' tuple[np.ndarray, np.ndarray | None, np.ndarray]: + """ + Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified + resolution, and optionally filter by classes. + + The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels + and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may + contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...] + (scaled to 0..999). For example: + + ``` + <|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|> + ``` + + Args: + result: String containing deepseek-vl2-formatted locations and labels. + resolution_wh: Tuple (width, height) to which we scale the box coordinates. + classes: Optional list of valid class names. If provided, boxes and labels not + in this list are filtered out. + + Returns: + xyxy (np.ndarray): An array of shape `(n, 4)` containing + the bounding boxes coordinates in format `[x1, y1, x2, y2]`. + class_id (Optional[np.ndarray]): An array of shape `(n,)` containing + the class indices for each bounding box (or `None` if classes is not + provided). + class_name (np.ndarray): An array of shape `(n,)` containing + the class labels for each bounding box. + """ # noqa: E501 + + width, height = resolution_wh + label_segments = re.findall(r"<\|ref\|>(.*?)<\|/ref\|>", result, flags=re.S) + detection_segments = re.findall(r"<\|det\|>(.*?)<\|/det\|>", result, flags=re.S) + + if len(label_segments) != len(detection_segments): + raise ValueError( + f"Number of ref tags ({len(label_segments)}) " + f"and det tags ({len(detection_segments)}) in the result must be equal." + ) + + xyxy, class_name_list = [], [] + for label, detection_blob in zip(label_segments, detection_segments): + current_class_name = label.strip() + for box in re.findall(r"\[(.*?)\]", detection_blob): + x1, y1, x2, y2 = map(float, box.strip("[]").split(",")) + xyxy.append( + [ + (x1 / 999 * width), + (y1 / 999 * height), + (x2 / 999 * width), + (y2 / 999 * height), + ] + ) + class_name_list.append(current_class_name) + + xyxy = np.array(xyxy, dtype=np.float32) + class_name = np.array(class_name_list) + + if classes is not None: + mask = np.array([name in classes for name in class_name], dtype=bool) + xyxy = xyxy[mask] + class_name = class_name[mask] + class_id = np.array([classes.index(name) for name in class_name]) + else: + unique_classes = sorted(list(set(class_name))) + class_to_id = {name: i for i, name in enumerate(unique_classes)} + class_id = np.array([class_to_id[name] for name in class_name]) + + return xyxy, class_id, class_name + + def from_florence_2( result: dict, resolution_wh: tuple[int, int] ) -> tuple[np.ndarray, np.ndarray | None, np.ndarray | None, np.ndarray | None]: diff --git a/test/detection/test_vlm.py b/test/detection/test_vlm.py index 1b4c2f18..8a8240e9 100644 --- a/test/detection/test_vlm.py +++ b/test/detection/test_vlm.py @@ -6,7 +6,10 @@ from contextlib import nullcontext as does_not_raise import numpy as np import pytest +from supervision.config import CLASS_NAME_DATA_FIELD +from supervision.detection.core import Detections from supervision.detection.vlm import ( + VLM, from_florence_2, from_google_gemini_2_0, from_google_gemini_2_5, @@ -1122,3 +1125,110 @@ def test_from_google_gemini_2_5( assert masks is not None assert masks.shape == expected_results[4].shape assert np.array_equal(masks, expected_results[4]) + + +@pytest.mark.parametrize( + "exception, result, resolution_wh, classes, expected_detections", + [ + ( + pytest.raises(ValueError), + "", + (100, 100), + None, + None, + ), # empty text + ( + pytest.raises(ValueError), + "random text", + (100, 100), + None, + None, + ), # random text + ( + does_not_raise(), + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + None, + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # single box, no classes + ( + does_not_raise(), + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + ["cat", "dog"], + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # single box, with classes + ( + does_not_raise(), + "<|ref|>person<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + ["cat", "dog"], + Detections.empty(), + ), # single box, wrong class + ( + does_not_raise(), + ( + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>" + "<|ref|>dog<|/ref|><|det|>[[500, 600, 700, 800]]<|/det|>" + ), + (1000, 1000), + ["cat"], + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # multiple boxes, one class correct + ( + pytest.raises(ValueError), + "<|ref|>cat<|/ref|>", + (100, 100), + None, + None, + ), # only ref + ( + pytest.raises(ValueError), + "<|det|>[[100, 200, 300, 400]]<|/det|>", + (100, 100), + None, + None, + ), # only det + ], +) +def test_from_deepseek_vl_2( + exception, + result: str, + resolution_wh: tuple[int, int], + classes: list[str] | None, + expected_detections: Detections, +): + with exception: + detections = Detections.from_vlm( + vlm=VLM.DEEPSEEK_VL_2, + result=result, + resolution_wh=resolution_wh, + classes=classes, + ) + + if expected_detections is None: + return + + assert len(detections) == len(expected_detections) + + if len(detections) == 0: + return + + assert np.allclose(detections.xyxy, expected_detections.xyxy, atol=1e-1) + assert np.array_equal(detections.class_id, expected_detections.class_id) + assert np.array_equal( + detections.data[CLASS_NAME_DATA_FIELD], + expected_detections.data[CLASS_NAME_DATA_FIELD], + )