xiaoxue_admin
/
supervision
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1884 from roboflow/feat/from_deepseek_vl_2_support 

feat: 🚀 from_vlm deepseek vl 2 support
This commit is contained in:
Piotr Skalski 2025-07-25 16:57:44 +02:00 committed by GitHub
parent 9679cd5c4e 8ca2e9fe8d
commit b381fa198f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 279 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
supervision/detection/core.py
View File

@ -40,6 +40,7 @@ from supervision.detection.utils.masks import calculate_masks_centroids
from supervision.detection.vlm import (
LMM,
VLM,
from_deepseek_vl_2,
from_florence_2,
from_google_gemini_2_0,
from_google_gemini_2_5,
@ -830,6 +831,7 @@ class Detections:
| Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` |
| Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` |
| Moondream | `MOONDREAM` | detection | `resolution_wh` | |
| DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` |
Args:
lmm (Union[LMM, str]): The type of LMM (Large Multimodal Model) to use.
@ -1117,6 +1119,47 @@ class Detections:
# array([[1752.28, 818.82, 2165.72, 1229.14],
# [1908.01, 1346.67, 2585.99, 2024.11]])
```
!!! example "DeepSeek-VL2"
??? tip "Prompt engineering"
To get the best results from DeepSeek-VL2, use optimized prompts that leverage
its object detection and visual grounding capabilities effectively.
**For general object detection, use the following user prompt:**
```
<image>\\n<|ref|>The giraffe at the front<|/ref|>
```
**For visual grounding, use the following user prompt:**
```
<image>\\n<|grounding|>Detect the giraffes
```
```python
from PIL import Image
import supervision as sv
deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>"
detections = sv.Detections.from_vlm(
vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size
)
detections.xyxy
# array([[ 420, 293, 724, 982],
# [ 18, 33, 458, 1084]])
detections.class_id
# array([0, 1])
detections.data
# {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype='<U24')}
```
""" # noqa: E501
# filler logic mapping old from_lmm to new from_vlm
@ -1124,6 +1167,7 @@ class Detections:
LMM.PALIGEMMA: VLM.PALIGEMMA,
LMM.FLORENCE_2: VLM.FLORENCE_2,
LMM.QWEN_2_5_VL: VLM.QWEN_2_5_VL,
LMM.DEEPSEEK_VL_2: VLM.DEEPSEEK_VL_2,
LMM.GOOGLE_GEMINI_2_0: VLM.GOOGLE_GEMINI_2_0,
LMM.GOOGLE_GEMINI_2_5: VLM.GOOGLE_GEMINI_2_5,
}
@ -1164,6 +1208,7 @@ class Detections:
| Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` |
| Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` |
| Moondream | `MOONDREAM` | detection | `resolution_wh` | |
| DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` |
Args:
vlm (Union[VLM, str]): The type of VLM (Vision Language Model) to use.
@ -1452,6 +1497,47 @@ class Detections:
# [1908.01, 1346.67, 2585.99, 2024.11]])
```
!!! example "DeepSeek-VL2"
??? tip "Prompt engineering"
To get the best results from DeepSeek-VL2, use optimized prompts that leverage
its object detection and visual grounding capabilities effectively.
**For general object detection, use the following user prompt:**
```
<image>\\n<|ref|>The giraffe at the front<|/ref|>
```
**For visual grounding, use the following user prompt:**
```
<image>\\n<|grounding|>Detect the giraffes
```
```python
from PIL import Image
import supervision as sv
deepseek_vl2_result = "<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>"
detections = sv.Detections.from_vlm(
vlm=sv.VLM.DEEPSEEK_VL_2, result=deepseek_vl2_result, resolution_wh=image.size
)
detections.xyxy
# array([[ 420, 293, 724, 982],
# [ 18, 33, 458, 1084]])
detections.class_id
# array([0, 1])
detections.data
# {'class_name': array(['The giraffe at the back', 'The giraffe at the front'], dtype='<U24')}
```
""" # noqa: E501
vlm = validate_vlm_parameters(vlm, result, kwargs)
@ -1466,6 +1552,11 @@ class Detections:
data = {CLASS_NAME_DATA_FIELD: class_name}
return cls(xyxy=xyxy, class_id=class_id, data=data)
if vlm == VLM.DEEPSEEK_VL_2:
xyxy, class_id, class_name = from_deepseek_vl_2(result, **kwargs)
data = {CLASS_NAME_DATA_FIELD: class_name}
return cls(xyxy=xyxy, class_id=class_id, data=data)
if vlm == VLM.FLORENCE_2:
xyxy, labels, mask, xyxyxyxy = from_florence_2(result, **kwargs)
if len(xyxy) == 0:

78
supervision/detection/vlm.py
View File

@ -36,6 +36,7 @@ class LMM(Enum):
PALIGEMMA = "paligemma"
FLORENCE_2 = "florence_2"
QWEN_2_5_VL = "qwen_2_5_vl"
DEEPSEEK_VL_2 = "deepseek_vl_2"
GOOGLE_GEMINI_2_0 = "gemini_2_0"
GOOGLE_GEMINI_2_5 = "gemini_2_5"
MOONDREAM = "moondream"
@ -76,6 +77,7 @@ class VLM(Enum):
PALIGEMMA = "paligemma"
FLORENCE_2 = "florence_2"
QWEN_2_5_VL = "qwen_2_5_vl"
DEEPSEEK_VL_2 = "deepseek_vl_2"
GOOGLE_GEMINI_2_0 = "gemini_2_0"
GOOGLE_GEMINI_2_5 = "gemini_2_5"
MOONDREAM = "moondream"
@ -104,6 +106,7 @@ RESULT_TYPES: dict[VLM, type] = {
VLM.PALIGEMMA: str,
VLM.FLORENCE_2: dict,
VLM.QWEN_2_5_VL: str,
VLM.DEEPSEEK_VL_2: str,
VLM.GOOGLE_GEMINI_2_0: str,
VLM.GOOGLE_GEMINI_2_5: str,
VLM.MOONDREAM: dict,
@ -113,6 +116,7 @@ REQUIRED_ARGUMENTS: dict[VLM, list[str]] = {
VLM.PALIGEMMA: ["resolution_wh"],
VLM.FLORENCE_2: ["resolution_wh"],
VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh"],
VLM.DEEPSEEK_VL_2: ["resolution_wh"],
VLM.GOOGLE_GEMINI_2_0: ["resolution_wh"],
VLM.GOOGLE_GEMINI_2_5: ["resolution_wh"],
VLM.MOONDREAM: ["resolution_wh"],
@ -122,6 +126,7 @@ ALLOWED_ARGUMENTS: dict[VLM, list[str]] = {
VLM.PALIGEMMA: ["resolution_wh", "classes"],
VLM.FLORENCE_2: ["resolution_wh"],
VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh", "classes"],
VLM.DEEPSEEK_VL_2: ["resolution_wh", "classes"],
VLM.GOOGLE_GEMINI_2_0: ["resolution_wh", "classes"],
VLM.GOOGLE_GEMINI_2_5: ["resolution_wh", "classes"],
VLM.MOONDREAM: ["resolution_wh"],
@ -310,6 +315,79 @@ def from_qwen_2_5_vl(
return xyxy, class_id, class_name
def from_deepseek_vl_2(
result: str, resolution_wh: tuple[int, int], classes: list[str] | None = None
) -> tuple[np.ndarray, np.ndarray | None, np.ndarray]:
"""
Parse bounding boxes from deepseek-vl2-formatted text, scale them to the specified
resolution, and optionally filter by classes.
The DeepSeek-VL2 output typically contains pairs of <|ref|> ... <|/ref|> labels
and <|det|> ... <|/det|> bounding box definitions. Each <|det|> section may
contain one or more bounding boxes in the form [[x1, y1, x2, y2], [x1, y1, x2, y2], ...]
(scaled to 0..999). For example:
```
<|ref|>The giraffe at the back<|/ref|><|det|>[[580, 270, 999, 904]]<|/det|><|ref|>The giraffe at the front<|/ref|><|det|>[[26, 31, 632, 998]]<|/det|><|end▁of▁sentence|>
```
Args:
result: String containing deepseek-vl2-formatted locations and labels.
resolution_wh: Tuple (width, height) to which we scale the box coordinates.
classes: Optional list of valid class names. If provided, boxes and labels not
in this list are filtered out.
Returns:
xyxy (np.ndarray): An array of shape `(n, 4)` containing
the bounding boxes coordinates in format `[x1, y1, x2, y2]`.
class_id (Optional[np.ndarray]): An array of shape `(n,)` containing
the class indices for each bounding box (or `None` if classes is not
provided).
class_name (np.ndarray): An array of shape `(n,)` containing
the class labels for each bounding box.
""" # noqa: E501
width, height = resolution_wh
label_segments = re.findall(r"<\|ref\|>(.*?)<\|/ref\|>", result, flags=re.S)
detection_segments = re.findall(r"<\|det\|>(.*?)<\|/det\|>", result, flags=re.S)
if len(label_segments) != len(detection_segments):
raise ValueError(
f"Number of ref tags ({len(label_segments)}) "
f"and det tags ({len(detection_segments)}) in the result must be equal."
)
xyxy, class_name_list = [], []
for label, detection_blob in zip(label_segments, detection_segments):
current_class_name = label.strip()
for box in re.findall(r"\[(.*?)\]", detection_blob):
x1, y1, x2, y2 = map(float, box.strip("[]").split(","))
xyxy.append(
[
(x1 / 999 * width),
(y1 / 999 * height),
(x2 / 999 * width),
(y2 / 999 * height),
]
)
class_name_list.append(current_class_name)
xyxy = np.array(xyxy, dtype=np.float32)
class_name = np.array(class_name_list)
if classes is not None:
mask = np.array([name in classes for name in class_name], dtype=bool)
xyxy = xyxy[mask]
class_name = class_name[mask]
class_id = np.array([classes.index(name) for name in class_name])
else:
unique_classes = sorted(list(set(class_name)))
class_to_id = {name: i for i, name in enumerate(unique_classes)}
class_id = np.array([class_to_id[name] for name in class_name])
return xyxy, class_id, class_name
def from_florence_2(
result: dict, resolution_wh: tuple[int, int]
) -> tuple[np.ndarray, np.ndarray | None, np.ndarray | None, np.ndarray | None]:

110
test/detection/test_vlm.py
View File

@ -6,7 +6,10 @@ from contextlib import nullcontext as does_not_raise
import numpy as np
import pytest
from supervision.config import CLASS_NAME_DATA_FIELD
from supervision.detection.core import Detections
from supervision.detection.vlm import (
VLM,
from_florence_2,
from_google_gemini_2_0,
from_google_gemini_2_5,
@ -1122,3 +1125,110 @@ def test_from_google_gemini_2_5(
assert masks is not None
assert masks.shape == expected_results[4].shape
assert np.array_equal(masks, expected_results[4])
@pytest.mark.parametrize(
"exception, result, resolution_wh, classes, expected_detections",
[
(
pytest.raises(ValueError),
"",
(100, 100),
None,
None,
), # empty text
(
pytest.raises(ValueError),
"random text",
(100, 100),
None,
None,
), # random text
(
does_not_raise(),
"<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>",
(1000, 1000),
None,
Detections(
xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]),
class_id=np.array([0]),
data={CLASS_NAME_DATA_FIELD: np.array(["cat"])},
),
), # single box, no classes
(
does_not_raise(),
"<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>",
(1000, 1000),
["cat", "dog"],
Detections(
xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]),
class_id=np.array([0]),
data={CLASS_NAME_DATA_FIELD: np.array(["cat"])},
),
), # single box, with classes
(
does_not_raise(),
"<|ref|>person<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>",
(1000, 1000),
["cat", "dog"],
Detections.empty(),
), # single box, wrong class
(
does_not_raise(),
(
"<|ref|>cat<|/ref|><|det|>[[100, 200, 300, 400]]<|/det|>"
"<|ref|>dog<|/ref|><|det|>[[500, 600, 700, 800]]<|/det|>"
),
(1000, 1000),
["cat"],
Detections(
xyxy=np.array([[100.1, 200.2, 300.3, 400.4]]),
class_id=np.array([0]),
data={CLASS_NAME_DATA_FIELD: np.array(["cat"])},
),
), # multiple boxes, one class correct
(
pytest.raises(ValueError),
"<|ref|>cat<|/ref|>",
(100, 100),
None,
None,
), # only ref
(
pytest.raises(ValueError),
"<|det|>[[100, 200, 300, 400]]<|/det|>",
(100, 100),
None,
None,
), # only det
],
)
def test_from_deepseek_vl_2(
exception,
result: str,
resolution_wh: tuple[int, int],
classes: list[str] | None,
expected_detections: Detections,
):
with exception:
detections = Detections.from_vlm(
vlm=VLM.DEEPSEEK_VL_2,
result=result,
resolution_wh=resolution_wh,
classes=classes,
)
if expected_detections is None:
return
assert len(detections) == len(expected_detections)
if len(detections) == 0:
return
assert np.allclose(detections.xyxy, expected_detections.xyxy, atol=1e-1)
assert np.array_equal(detections.class_id, expected_detections.class_id)
assert np.array_equal(
detections.data[CLASS_NAME_DATA_FIELD],
expected_detections.data[CLASS_NAME_DATA_FIELD],
)