From f737782ae848a5437d7210c8cf4cecfc93a2ad90 Mon Sep 17 00:00:00 2001 From: SkalskiP Date: Mon, 7 Apr 2025 14:56:58 +0200 Subject: [PATCH 01/11] initial commit --- supervision/detection/core.py | 42 ++++++++++++++++ supervision/detection/vlm.py | 92 +++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index f508dd62..9cda122a 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -40,6 +40,7 @@ from supervision.detection.vlm import ( from_paligemma, from_qwen_2_5_vl, validate_vlm_parameters, + from_deepseek_vl_2, ) from supervision.geometry.core import Position from supervision.utils.internal import deprecated, get_instance_variables @@ -849,6 +850,7 @@ class Detections: LMM.PALIGEMMA: VLM.PALIGEMMA, LMM.FLORENCE_2: VLM.FLORENCE_2, LMM.QWEN_2_5_VL: VLM.QWEN_2_5_VL, + LMM.DEEPSEEK_VL_2: VLM.DEEPSEEK_VL_2 } # (this works even if the LMM enum is wrapped by @deprecated) @@ -876,6 +878,41 @@ class Detections: def from_vlm( cls, vlm: Union[VLM, str], result: Union[str, dict], **kwargs: Any ) -> Detections: + """ + Creates a Detections object from the given result string based on the specified + Vision-Language Model (LMM). + + Args: + vlm (Union[VLM, str]): The type of VLM (Vision-Language Model) to use. + result (str): The result string containing the detection data. + **kwargs (Any): Additional keyword arguments required by the specified VLM. + + Returns: + Detections: A new Detections object. + + Raises: + ValueError: If the LMM is invalid, required arguments are missing, or + disallowed arguments are provided. + ValueError: If the specified LMM is not supported. + + Examples: + ```python + import supervision as sv + + paligemma_result = " cat" + detections = sv.Detections.from_vlm( + sv.VLM.PALIGEMMA, + paligemma_result, + resolution_wh=(1000, 1000), + classes=['cat', 'dog'] + ) + detections.xyxy + # array([[250., 250., 750., 750.]]) + + detections.class_id + # array([0]) + ``` + """ vlm = validate_vlm_parameters(vlm, result, kwargs) if vlm == VLM.PALIGEMMA: @@ -888,6 +925,11 @@ class Detections: data = {CLASS_NAME_DATA_FIELD: class_name} return cls(xyxy=xyxy, class_id=class_id, data=data) + if vlm == VLM.DEEPSEEK_VL_2: + xyxy, class_id, class_name = from_deepseek_vl_2(result, **kwargs) + data = {CLASS_NAME_DATA_FIELD: class_name} + return cls(xyxy=xyxy, class_id=class_id, data=data) + if vlm == VLM.FLORENCE_2: xyxy, labels, mask, xyxyxyxy = from_florence_2(result, **kwargs) if len(xyxy) == 0: diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index 719dc443..e9b72bef 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -1,5 +1,6 @@ import json import re +import ast from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union @@ -17,30 +18,35 @@ class LMM(Enum): PALIGEMMA = "paligemma" FLORENCE_2 = "florence_2" QWEN_2_5_VL = "qwen_2_5_vl" + DEEPSEEK_VL_2 = "deepseek_vl_2" class VLM(Enum): PALIGEMMA = "paligemma" FLORENCE_2 = "florence_2" QWEN_2_5_VL = "qwen_2_5_vl" + DEEPSEEK_VL_2 = "deepseek_vl_2" RESULT_TYPES: Dict[VLM, type] = { VLM.PALIGEMMA: str, VLM.FLORENCE_2: dict, VLM.QWEN_2_5_VL: str, + VLM.DEEPSEEK_VL_2: str, } REQUIRED_ARGUMENTS: Dict[VLM, List[str]] = { VLM.PALIGEMMA: ["resolution_wh"], VLM.FLORENCE_2: ["resolution_wh"], VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh"], + VLM.DEEPSEEK_VL_2: ["resolution_wh"], } ALLOWED_ARGUMENTS: Dict[VLM, List[str]] = { VLM.PALIGEMMA: ["resolution_wh", "classes"], VLM.FLORENCE_2: ["resolution_wh"], VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh", "classes"], + VLM.DEEPSEEK_VL_2: ["resolution_wh", "classes"], } SUPPORTED_TASKS_FLORENCE_2 = [ @@ -223,6 +229,92 @@ def from_qwen_2_5_vl( return xyxy, class_id, class_name +def from_deepseek_vl_2( + result: str, + resolution_wh: Tuple[int, int], + classes: Optional[List[str]] = None +) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]: + """ + Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified + resolution, and optionally filter by classes. + + The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels + and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may + contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...] + (scaled to 0..999). However, other text (e.g. <｜end▁of▁sentence｜>) may appear + after the bracket, so we strip that out here. + + Args: + result: String containing deepseek-vl2-formatted locations and labels. + resolution_wh: Tuple (width, height) to which we scale the box coordinates. + classes: Optional list of valid class names. If provided, boxes and labels not + in this list are filtered out. + + Returns: + xyxy (np.ndarray): An array of shape `(n, 4)` containing + the bounding boxes coordinates in format `[x1, y1, x2, y2]`. + class_id (Optional[np.ndarray]): An array of shape `(n,)` containing + the class indices for each bounding box (or `None` if classes is not + provided). + class_name (np.ndarray): An array of shape `(n,)` containing + the class labels for each bounding box. + """ + + w, h = resolution_wh + if w <= 0 or h <= 0: + raise ValueError( + f"Both dimensions in resolution_wh must be positive. Got ({w}, {h})." + ) + + label_segments = re.findall(r'<\|ref\|>(.*?)<\|/ref\|>', result, flags=re.DOTALL) + bbox_segments = re.findall(r'<\|det\|>(.*?)<\|/det\|>', result, flags=re.DOTALL) + + if len(label_segments) != len(bbox_segments): + return np.empty((0, 4)), None, np.empty((0,), dtype=str) + + boxes, labels = [], [] + + for label_str, bbox_str in zip(label_segments, bbox_segments): + label_str = label_str.strip() + raw_box_groups = re.findall(r'\[\[.*?\]\]', bbox_str, flags=re.DOTALL) + + if not raw_box_groups: + continue + + for group_str in raw_box_groups: + try: + list_of_boxes = ast.literal_eval(group_str) + for box in list_of_boxes: + if len(box) != 4: + continue + + x1 = box[0] / 999.0 * w + y1 = box[1] / 999.0 * h + x2 = box[2] / 999.0 * w + y2 = box[3] / 999.0 * h + + boxes.append([x1, y1, x2, y2]) + labels.append(label_str) + + except (SyntaxError, ValueError): + continue + + if len(boxes) == 0: + return np.empty((0, 4)), None, np.empty((0,), dtype=str) + + xyxy = np.array(boxes, dtype=np.float32) + class_name = np.array(labels, dtype=str) + class_id = None + + if classes is not None: + mask = np.array([name in classes for name in class_name], dtype=bool) + xyxy = xyxy[mask] + class_name = class_name[mask] + class_id = np.array([classes.index(name) for name in class_name]) + + return xyxy, class_id, class_name + + def from_florence_2( result: dict, resolution_wh: Tuple[int, int] ) -> Tuple[ From 8e845caeeb804b21776f31ebbc4cba9aa3108f1b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 15:44:09 +0000 Subject: [PATCH 02/11] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/detection/core.py | 3 +-- supervision/detection/vlm.py | 12 +++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 06a2f466..ca821592 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -36,12 +36,12 @@ from supervision.detection.utils import ( from supervision.detection.vlm import ( LMM, VLM, + from_deepseek_vl_2, from_florence_2, from_google_gemini, from_paligemma, from_qwen_2_5_vl, validate_vlm_parameters, - from_deepseek_vl_2, ) from supervision.geometry.core import Position from supervision.utils.internal import deprecated, get_instance_variables @@ -902,7 +902,6 @@ class Detections: LMM.DEEPSEEK_VL_2: VLM.DEEPSEEK_VL_2, LMM.GOOGLE_GEMINI_2_0: VLM.GOOGLE_GEMINI_2_0, LMM.GOOGLE_GEMINI_2_5: VLM.GOOGLE_GEMINI_2_5, - } # (this works even if the LMM enum is wrapped by @deprecated) diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index f8b45d48..35c0c68e 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -1,6 +1,6 @@ +import ast import json import re -import ast from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union @@ -244,9 +244,7 @@ def from_qwen_2_5_vl( def from_deepseek_vl_2( - result: str, - resolution_wh: Tuple[int, int], - classes: Optional[List[str]] = None + result: str, resolution_wh: Tuple[int, int], classes: Optional[List[str]] = None ) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]: """ Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified @@ -280,8 +278,8 @@ def from_deepseek_vl_2( f"Both dimensions in resolution_wh must be positive. Got ({w}, {h})." ) - label_segments = re.findall(r'<\|ref\|>(.*?)<\|/ref\|>', result, flags=re.DOTALL) - bbox_segments = re.findall(r'<\|det\|>(.*?)<\|/det\|>', result, flags=re.DOTALL) + label_segments = re.findall(r"<\|ref\|>(.*?)<\|/ref\|>", result, flags=re.DOTALL) + bbox_segments = re.findall(r"<\|det\|>(.*?)<\|/det\|>", result, flags=re.DOTALL) if len(label_segments) != len(bbox_segments): return np.empty((0, 4)), None, np.empty((0,), dtype=str) @@ -290,7 +288,7 @@ def from_deepseek_vl_2( for label_str, bbox_str in zip(label_segments, bbox_segments): label_str = label_str.strip() - raw_box_groups = re.findall(r'\[\[.*?\]\]', bbox_str, flags=re.DOTALL) + raw_box_groups = re.findall(r"\[\[.*?\]\]", bbox_str, flags=re.DOTALL) if not raw_box_groups: continue From 535cb2a91b2c54bf56704435a96721bc1ce8070f Mon Sep 17 00:00:00 2001 From: Onuralp SEZER Date: Fri, 11 Jul 2025 18:49:50 +0300 Subject: [PATCH 03/11] =?UTF-8?q?fix(vlm):=20=F0=9F=93=9D=20correct=20form?= =?UTF-8?q?atting=20in=20docstring=20for=20from=5Fdeepseek=5Fvl=5F2=20func?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/detection/vlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index 35c0c68e..bf5c116b 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -253,7 +253,7 @@ def from_deepseek_vl_2( The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...] - (scaled to 0..999). However, other text (e.g. <｜end▁of▁sentence｜>) may appear + (scaled to 0..999). However, other text (e.g. < | end▁of▁sentence | >) may appear after the bracket, so we strip that out here. Args: @@ -270,7 +270,7 @@ def from_deepseek_vl_2( provided). class_name (np.ndarray): An array of shape `(n,)` containing the class labels for each bounding box. - """ + """ # noqa: E501 w, h = resolution_wh if w <= 0 or h <= 0: From a470bc29769c710b57a453215fe5fe17e48b4382 Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Wed, 23 Jul 2025 18:36:31 +0530 Subject: [PATCH 04/11] chore: make pre-commit happy --- supervision/detection/vlm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index f0e79ae4..24d542b2 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -317,8 +317,8 @@ def from_qwen_2_5_vl( def from_deepseek_vl_2( - result: str, resolution_wh: Tuple[int, int], classes: Optional[List[str]] = None -) -> Tuple[np.ndarray, Optional[np.ndarray], np.ndarray]: + result: str, resolution_wh: tuple[int, int], classes: list[str] | None = None +) -> tuple[np.ndarray, np.ndarray | None, np.ndarray]: """ Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified resolution, and optionally filter by classes. From b3aee43c3be09906355ade76821ea6adf7d42373 Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Thu, 24 Jul 2025 12:55:40 +0530 Subject: [PATCH 05/11] update: from_deepseek_vl_2 logic --- supervision/detection/vlm.py | 82 +++++++++++++++--------------------- 1 file changed, 34 insertions(+), 48 deletions(-) diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index 24d542b2..ffc0d779 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ast import base64 import io import json @@ -326,8 +325,11 @@ def from_deepseek_vl_2( The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...] - (scaled to 0..999). However, other text (e.g. < | end▁of▁sentence | >) may appear - after the bracket, so we strip that out here. + (scaled to 0..999). For example: + + ``` + <|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|> + ``` Args: result: String containing deepseek-vl2-formatted locations and labels. @@ -345,59 +347,43 @@ def from_deepseek_vl_2( the class labels for each bounding box. """ # noqa: E501 - w, h = resolution_wh - if w <= 0 or h <= 0: + width, height = resolution_wh + label_segments = re.findall(r"<\|ref\|>(.*?)<\|/ref\|>", result, flags=re.S) + detection_segments = re.findall(r"<\|det\|>(.*?)<\|/det\|>", result, flags=re.S) + + if len(label_segments) != len(detection_segments): raise ValueError( - f"Both dimensions in resolution_wh must be positive. Got ({w}, {h})." + f"Number of ref tags ({len(label_segments)}) " + f"and det tags ({len(detection_segments)}) in the result must be equal." ) - label_segments = re.findall(r"<\|ref\|>(.*?)<\|/ref\|>", result, flags=re.DOTALL) - bbox_segments = re.findall(r"<\|det\|>(.*?)<\|/det\|>", result, flags=re.DOTALL) + xyxy, class_names = [], [] + for label, detection_blob in zip(label_segments, detection_segments): + class_name = label.strip() + for box in re.findall(r"\[(.*?)\]", detection_blob): + x1, y1, x2, y2 = map(float, box.strip("[]").split(",")) + xyxy.append( + [ + int(x1 / 999 * width), + int(y1 / 999 * height), + int(x2 / 999 * width), + int(y2 / 999 * height), + ] + ) + class_names.append(class_name) - if len(label_segments) != len(bbox_segments): - return np.empty((0, 4)), None, np.empty((0,), dtype=str) - - boxes, labels = [], [] - - for label_str, bbox_str in zip(label_segments, bbox_segments): - label_str = label_str.strip() - raw_box_groups = re.findall(r"\[\[.*?\]\]", bbox_str, flags=re.DOTALL) - - if not raw_box_groups: - continue - - for group_str in raw_box_groups: - try: - list_of_boxes = ast.literal_eval(group_str) - for box in list_of_boxes: - if len(box) != 4: - continue - - x1 = box[0] / 999.0 * w - y1 = box[1] / 999.0 * h - x2 = box[2] / 999.0 * w - y2 = box[3] / 999.0 * h - - boxes.append([x1, y1, x2, y2]) - labels.append(label_str) - - except (SyntaxError, ValueError): - continue - - if len(boxes) == 0: - return np.empty((0, 4)), None, np.empty((0,), dtype=str) - - xyxy = np.array(boxes, dtype=np.float32) - class_name = np.array(labels, dtype=str) - class_id = None + xyxy = np.array(xyxy) + class_names = np.array(class_names) if classes is not None: - mask = np.array([name in classes for name in class_name], dtype=bool) + mask = np.array([name in classes for name in class_names], dtype=bool) xyxy = xyxy[mask] - class_name = class_name[mask] - class_id = np.array([classes.index(name) for name in class_name]) + class_names = class_names[mask] + class_id = np.array([classes.index(name) for name in class_names]) + else: + class_id = np.array(list(range(len(class_names)))) - return xyxy, class_id, class_name + return xyxy, class_id, class_names def from_florence_2( From cc1525707aaa5bf81c9ae076a1f44c9118a40bec Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Thu, 24 Jul 2025 13:11:06 +0530 Subject: [PATCH 06/11] update: docstring for Detections.from_vlm --- supervision/detection/core.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index de665d28..0233ee3f 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -1166,6 +1166,7 @@ class Detections: | Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` | | Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` | | Moondream | `MOONDREAM` | detection | `resolution_wh` | | + | DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` | Args: vlm (Union[VLM, str]): The type of VLM (Vision Language Model) to use. @@ -1454,6 +1455,29 @@ class Detections: # [1908.01, 1346.67, 2585.99, 2024.11]]) ``` + !!! example "DeepSeek-VL2" + + ```python + from PIL import Image + import supervision as sv + + deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>" + + detections = sv.Detections.from_vlm( + vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size + ) + + detections.xyxy + # array([[ 420, 293, 724, 982], + # [ 18, 33, 458, 1084]]) + + detections.class_id + # array([0, 1]) + + detections.data + # {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype=' Date: Thu, 24 Jul 2025 14:23:24 +0530 Subject: [PATCH 07/11] add: prompt engineering tip for deepseek-vl2 --- supervision/detection/core.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 0233ee3f..65429fd6 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -831,6 +831,7 @@ class Detections: | Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` | | Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` | | Moondream | `MOONDREAM` | detection | `resolution_wh` | | + | DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` | Args: lmm (Union[LMM, str]): The type of LMM (Large Multimodal Model) to use. @@ -1457,6 +1458,24 @@ class Detections: !!! example "DeepSeek-VL2" + + ??? tip "Prompt engineering" + + To get the best results from DeepSeek-VL2, use optimized prompts that leverage + its object detection and visual grounding capabilities effectively. + + **For general object detection, use the following user prompt:** + + ``` + \\n<|ref|>The giraffe at the front<|/ref|> + ``` + + **For visual grounding, use the following user prompt:** + + ``` + \\n<|grounding|>Detect the giraffes + ``` + ```python from PIL import Image import supervision as sv From 8fa21dd165a99f34cbc24c60d127ca3a5881303f Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Thu, 24 Jul 2025 15:04:59 +0530 Subject: [PATCH 08/11] update: from_deepseek_vl_2 --- supervision/detection/vlm.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py index ffc0d779..71207554 100644 --- a/supervision/detection/vlm.py +++ b/supervision/detection/vlm.py @@ -357,33 +357,35 @@ def from_deepseek_vl_2( f"and det tags ({len(detection_segments)}) in the result must be equal." ) - xyxy, class_names = [], [] + xyxy, class_name_list = [], [] for label, detection_blob in zip(label_segments, detection_segments): - class_name = label.strip() + current_class_name = label.strip() for box in re.findall(r"\[(.*?)\]", detection_blob): x1, y1, x2, y2 = map(float, box.strip("[]").split(",")) xyxy.append( [ - int(x1 / 999 * width), - int(y1 / 999 * height), - int(x2 / 999 * width), - int(y2 / 999 * height), + (x1 / 999 * width), + (y1 / 999 * height), + (x2 / 999 * width), + (y2 / 999 * height), ] ) - class_names.append(class_name) + class_name_list.append(current_class_name) - xyxy = np.array(xyxy) - class_names = np.array(class_names) + xyxy = np.array(xyxy, dtype=np.float32) + class_name = np.array(class_name_list) if classes is not None: - mask = np.array([name in classes for name in class_names], dtype=bool) + mask = np.array([name in classes for name in class_name], dtype=bool) xyxy = xyxy[mask] - class_names = class_names[mask] - class_id = np.array([classes.index(name) for name in class_names]) + class_name = class_name[mask] + class_id = np.array([classes.index(name) for name in class_name]) else: - class_id = np.array(list(range(len(class_names)))) + unique_classes = sorted(list(set(class_name))) + class_to_id = {name: i for i, name in enumerate(unique_classes)} + class_id = np.array([class_to_id[name] for name in class_name]) - return xyxy, class_id, class_names + return xyxy, class_id, class_name def from_florence_2( From 137078602ba2a1eb845c4131e62f68bc2a6a0cbb Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Thu, 24 Jul 2025 15:17:05 +0530 Subject: [PATCH 09/11] add: tests --- test/detection/test_vlm.py | 110 +++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/test/detection/test_vlm.py b/test/detection/test_vlm.py index 1b4c2f18..8a8240e9 100644 --- a/test/detection/test_vlm.py +++ b/test/detection/test_vlm.py @@ -6,7 +6,10 @@ from contextlib import nullcontext as does_not_raise import numpy as np import pytest +from supervision.config import CLASS_NAME_DATA_FIELD +from supervision.detection.core import Detections from supervision.detection.vlm import ( + VLM, from_florence_2, from_google_gemini_2_0, from_google_gemini_2_5, @@ -1122,3 +1125,110 @@ def test_from_google_gemini_2_5( assert masks is not None assert masks.shape == expected_results[4].shape assert np.array_equal(masks, expected_results[4]) + + +@pytest.mark.parametrize( + "exception, result, resolution_wh, classes, expected_detections", + [ + ( + pytest.raises(ValueError), + "", + (100, 100), + None, + None, + ), # empty text + ( + pytest.raises(ValueError), + "random text", + (100, 100), + None, + None, + ), # random text + ( + does_not_raise(), + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + None, + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # single box, no classes + ( + does_not_raise(), + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + ["cat", "dog"], + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # single box, with classes + ( + does_not_raise(), + "<|ref|>person<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>", + (1000, 1000), + ["cat", "dog"], + Detections.empty(), + ), # single box, wrong class + ( + does_not_raise(), + ( + "<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>" + "<|ref|>dog<|/ref|><|det|>[[500, 600, 700, 800]]<|/det|>" + ), + (1000, 1000), + ["cat"], + Detections( + xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]), + class_id=np.array([0]), + data={CLASS_NAME_DATA_FIELD: np.array(["cat"])}, + ), + ), # multiple boxes, one class correct + ( + pytest.raises(ValueError), + "<|ref|>cat<|/ref|>", + (100, 100), + None, + None, + ), # only ref + ( + pytest.raises(ValueError), + "<|det|>[[100, 200, 300, 400]]<|/det|>", + (100, 100), + None, + None, + ), # only det + ], +) +def test_from_deepseek_vl_2( + exception, + result: str, + resolution_wh: tuple[int, int], + classes: list[str] | None, + expected_detections: Detections, +): + with exception: + detections = Detections.from_vlm( + vlm=VLM.DEEPSEEK_VL_2, + result=result, + resolution_wh=resolution_wh, + classes=classes, + ) + + if expected_detections is None: + return + + assert len(detections) == len(expected_detections) + + if len(detections) == 0: + return + + assert np.allclose(detections.xyxy, expected_detections.xyxy, atol=1e-1) + assert np.array_equal(detections.class_id, expected_detections.class_id) + assert np.array_equal( + detections.data[CLASS_NAME_DATA_FIELD], + expected_detections.data[CLASS_NAME_DATA_FIELD], + ) From bc1d73a30721471472e3e06ec03cac1f419e02c1 Mon Sep 17 00:00:00 2001 From: soumik12345 <19soumik.rakshit96@gmail.com> Date: Fri, 25 Jul 2025 20:13:28 +0530 Subject: [PATCH 10/11] chore: address feedback --- supervision/detection/core.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 65429fd6..3865f5d7 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -1119,6 +1119,47 @@ class Detections: # array([[1752.28, 818.82, 2165.72, 1229.14], # [1908.01, 1346.67, 2585.99, 2024.11]]) ``` + + !!! example "DeepSeek-VL2" + + + ??? tip "Prompt engineering" + + To get the best results from DeepSeek-VL2, use optimized prompts that leverage + its object detection and visual grounding capabilities effectively. + + **For general object detection, use the following user prompt:** + + ``` + \\n<|ref|>The giraffe at the front<|/ref|> + ``` + + **For visual grounding, use the following user prompt:** + + ``` + \\n<|grounding|>Detect the giraffes + ``` + + ```python + from PIL import Image + import supervision as sv + + deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>" + + detections = sv.Detections.from_vlm( + vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size + ) + + detections.xyxy + # array([[ 420, 293, 724, 982], + # [ 18, 33, 458, 1084]]) + + detections.class_id + # array([0, 1]) + + detections.data + # {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype=' Date: Fri, 25 Jul 2025 14:44:10 +0000 Subject: [PATCH 11/11] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- supervision/detection/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supervision/detection/core.py b/supervision/detection/core.py index 3865f5d7..ffe5ed3f 100644 --- a/supervision/detection/core.py +++ b/supervision/detection/core.py @@ -1119,7 +1119,7 @@ class Detections: # array([[1752.28, 818.82, 2165.72, 1229.14], # [1908.01, 1346.67, 2585.99, 2024.11]]) ``` - + !!! example "DeepSeek-VL2"