xiaoxue_admin
/
supervision
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(vlm): handle malformed Gemini/Qwen model output without crashing (#2342) 

Two ways the VLM parsers crashed on adversarial model output instead of
degrading gracefully (the contract they already honor for invalid JSON):

1. Gemini 2.5: a mask value that is not a 'data:image/png;base64,' string
   appended an empty mask and then 'continue'd, skipping the confidence
   handler at the bottom of the loop. The item's box was recorded but its
   confidence was not, so the confidence array ended up shorter than xyxy
   and Detections.from_vlm raised a shape ValueError. Replaced the
   'continue' with an if/else so the confidence handler always runs.

2. Gemini 2.0 / Gemini 2.5 / Qwen 2.5: valid JSON whose top level is not a
   list, or whose elements are not dicts (e.g. '[1, 2, 3]'), raised
   TypeError from the 'key not in item' membership test. Added a top-level
   list guard (Gemini 2.0/2.5; Qwen already had one) and a per-element
   dict guard so wrong-shaped JSON degrades to empty Detections.

Add regression tests for the mask/confidence alignment and for graceful
degradation across all three parsers.

---------

Co-authored-by: jirka <6035284+Borda@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
This commit is contained in:
Ruben 2026-06-18 15:31:31 +02:00 committed by GitHub
parent 4b60bbc9cc
commit 44546a13f2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 111 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/changelog.md
View File

@ -15,6 +15,8 @@ date_modified: 2026-06-16
- Fixed [#2322](https://github.com/roboflow/supervision/pull/2322): COCO export now preserves all polygon parts for multi-component masks. Previously, only the first polygon was written when a non-crowd mask had disjoint segments; all parts are now included.
- Fixed [#2342](https://github.com/roboflow/supervision/pull/2342): `sv.Detections.from_vlm` with `sv.VLM.GOOGLE_GEMINI_2_0`, `sv.VLM.GOOGLE_GEMINI_2_5`, and `sv.VLM.QWEN_2_5_VL` no longer raises when the model returns valid JSON of the wrong shape (non-list top-level or non-dict elements). A non-string or malformed `"mask"` value in Gemini 2.5 output no longer triggers `AttributeError`; invalid base64 or non-PNG mask data falls back to an empty mask, keeping `xyxy`, `confidence`, and `masks` arrays aligned.
- Fixed [#2333](https://github.com/roboflow/supervision/pull/2333): [`sv.DetectionsSmoother`](https://supervision.roboflow.com/latest/detection/tools/smoother/#supervision.detection.tools.smoother.DetectionsSmoother) no longer raises when smoothing detections without `confidence`. Confidence is now averaged over the frames that carry it; when tracks in the same frame disagree on confidence presence, `confidence` is set to `None` for all smoothed detections.
- Fixed [#2341](https://github.com/roboflow/supervision/pull/2341): `sv.DetectionDataset.as_pascal_voc` no longer mutates the source `Detections.xyxy` by the 1-index offset on every call. Previously, repeated exports accumulated a `+1` shift in the caller's bounding boxes.

103
src/supervision/detection/vlm.py
View File

@ -214,7 +214,7 @@ def validate_vlm_parameters(vlm: VLM | str, result: Any, kwargs: dict[str, Any])
def from_paligemma(
result: str, resolution_wh: tuple[int, int], classes: list[str] | None = None
) -> tuple[npt.NDArray[Any], npt.NDArray[Any], npt.NDArray[Any]]:
) -> tuple[npt.NDArray[Any], npt.NDArray[Any] | None, npt.NDArray[Any]]:
"""
Parse bounding boxes from paligemma-formatted text, scale them to the specified
resolution, and optionally filter by classes.
@ -238,23 +238,23 @@ def from_paligemma(
r"(?<!<loc\d{4}>)<loc(\d{4})><loc(\d{4})><loc(\d{4})><loc(\d{4})> ([\w\s\-]+)"
)
matches = pattern.findall(result)
matches = np.array(matches) if matches else np.empty((0, 5))
matches_arr: npt.NDArray[Any] = np.array(matches) if matches else np.empty((0, 5))
if matches.shape[0] == 0:
if matches_arr.shape[0] == 0:
return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty(0, dtype=str)
xyxy, class_name = matches[:, [1, 0, 3, 2]], matches[:, 4]
xyxy = xyxy.astype(int) / 1024 * np.array([w, h, w, h])
class_name = np.char.strip(class_name.astype(str))
class_id = None
xyxy_arr = np.array(matches_arr[:, [1, 0, 3, 2]], dtype=float)
xyxy_arr = xyxy_arr.astype(int) / 1024 * np.array([w, h, w, h])
class_name = np.char.strip(matches_arr[:, 4].astype(str))
class_id: npt.NDArray[Any] | None = None
if classes is not None:
mask = np.array([name in classes for name in class_name], dtype=bool)
xyxy = xyxy[mask]
xyxy_arr = xyxy_arr[mask]
class_name = class_name[mask]
class_id = np.array([classes.index(name) for name in class_name])
return xyxy, class_id, class_name
return xyxy_arr, class_id, class_name
def recover_truncated_qwen_2_5_vl_response(text: str) -> Any | None:
@ -369,7 +369,7 @@ def from_qwen_2_5_vl(
labels_list = []
for item in data:
if "bbox_2d" not in item or "label" not in item:
if not isinstance(item, dict) or "bbox_2d" not in item or "label" not in item:
continue
boxes_list.append(item["bbox_2d"])
labels_list.append(item["label"])
@ -460,12 +460,13 @@ def from_deepseek_vl_2(
f"and det tags ({len(detection_segments)}) in the result must be equal."
)
xyxy, class_name_list = [], []
xyxy_list: list[list[float]] = []
class_name_list: list[str] = []
for label, detection_blob in zip(label_segments, detection_segments):
current_class_name = label.strip()
for box in re.findall(r"\[(.*?)\]", detection_blob):
x1, y1, x2, y2 = map(float, box.strip("[]").split(","))
xyxy.append(
xyxy_list.append(
[
(x1 / 999 * width),
(y1 / 999 * height),
@ -475,7 +476,7 @@ def from_deepseek_vl_2(
)
class_name_list.append(current_class_name)
xyxy = np.array(xyxy, dtype=np.float32)
xyxy = np.array(xyxy_list, dtype=np.float32)
class_name = np.array(class_name_list)
if classes is not None:
@ -541,15 +542,15 @@ def from_florence_2(
return xyxy, labels, None, xyxyxyxy
if task in ["<REFERRING_EXPRESSION_SEGMENTATION>", "<REGION_TO_SEGMENTATION>"]:
xyxy_list = []
masks_list = []
xyxy_list: list[npt.NDArray[Any]] = []
masks_list: list[npt.NDArray[Any]] = []
for polygons_of_same_class in result["polygons"]:
for polygon in polygons_of_same_class:
polygon = np.reshape(polygon, (-1, 2)).astype(np.int32)
mask = polygon_to_mask(polygon, resolution_wh).astype(bool)
masks_list.append(mask)
xyxy = polygon_to_xyxy(polygon)
xyxy_list.append(xyxy)
xyxy_box = polygon_to_xyxy(polygon)
xyxy_list.append(xyxy_box)
# per-class labels also provided, but they are ["", "", "", ...]
# when we figure out how to set class names, we can do
# zip(result["labels"], result["polygons"])
@ -640,11 +641,14 @@ def from_google_gemini_2_0(
except json.JSONDecodeError:
return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str)
if not isinstance(data, list):
return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str)
labels = []
xyxy = []
for item in data:
if "box_2d" not in item or "label" not in item:
if not isinstance(item, dict) or "box_2d" not in item or "label" not in item:
continue
labels.append(item["label"])
box = item["box_2d"]
@ -734,13 +738,22 @@ def from_google_gemini_2_5(
None,
)
if not isinstance(data, list):
return (
np.empty((0, 4)),
np.array([], dtype=int),
np.array([], dtype=str),
np.array([], dtype=float),
None,
)
boxes_list: list[Any] = []
labels_list: list[str] = []
confidence_list: list[float] | None = []
masks_list: list[npt.NDArray[Any]] | None = []
for item in data:
if "box_2d" not in item or "label" not in item:
if not isinstance(item, dict) or "box_2d" not in item or "label" not in item:
continue
labels_list.append(item["label"])
box = item["box_2d"]
@ -755,29 +768,39 @@ def from_google_gemini_2_5(
if "mask" in item:
if masks_list is not None:
png_str = item["mask"]
if not png_str.startswith("data:image/png;base64,"):
if not isinstance(png_str, str) or not png_str.startswith(
"data:image/png;base64,"
):
# Malformed mask: keep an empty mask but still fall through to
# the confidence handling below, so the per-item arrays stay
# aligned (a `continue` here desynced confidence vs boxes).
masks_list.append(np.zeros((h, w), dtype=bool))
continue
png_str = png_str.removeprefix("data:image/png;base64,")
png_str = base64.b64decode(png_str)
mask_img = Image.open(io.BytesIO(png_str))
y_min, y_max = int(absolute_bbox[1]), int(absolute_bbox[3])
x_min, x_max = int(absolute_bbox[0]), int(absolute_bbox[2])
bbox_height = y_max - y_min
bbox_width = x_max - x_min
if bbox_height > 0 and bbox_width > 0:
mask_img = mask_img.resize(
(bbox_width, bbox_height), resample=Image.Resampling.BILINEAR
)
np_mask: npt.NDArray[np.bool_] = np.zeros((h, w), dtype=bool)
np_mask[y_min:y_max, x_min:x_max] = np.array(mask_img) > 0
masks_list.append(np_mask)
else:
masks_list.append(np.zeros((h, w), dtype=bool))
png_str = png_str.removeprefix("data:image/png;base64,")
try:
png_bytes = base64.b64decode(png_str)
mask_img = Image.open(io.BytesIO(png_bytes)).convert("L")
except Exception:
masks_list.append(np.zeros((h, w), dtype=bool))
else:
y_min, y_max = int(absolute_bbox[1]), int(absolute_bbox[3])
x_min, x_max = int(absolute_bbox[0]), int(absolute_bbox[2])
bbox_height = y_max - y_min
bbox_width = x_max - x_min
if bbox_height > 0 and bbox_width > 0:
mask_img = mask_img.resize(
(bbox_width, bbox_height),
resample=Image.Resampling.BILINEAR,
)
np_mask: npt.NDArray[np.bool_] = np.zeros(
(h, w), dtype=bool
)
np_mask[y_min:y_max, x_min:x_max] = np.array(mask_img) > 0
masks_list.append(np_mask)
else:
masks_list.append(np.zeros((h, w), dtype=bool))
else:
masks_list = None

46
tests/detection/test_vlm.py
View File

@ -1297,3 +1297,49 @@ def test_from_deepseek_vl_2(
detections.data[CLASS_NAME_DATA_FIELD],
expected_detections.data[CLASS_NAME_DATA_FIELD],
)
def test_from_google_gemini_2_5_malformed_mask_keeps_confidence_aligned():
"""A non-data-URI mask must not skip the item's confidence and desync arrays."""
result = (
'[{"box_2d": [10, 10, 100, 100], "label": "cat", "mask": "bad", '
'"confidence": 0.8}, {"box_2d": [20, 20, 120, 120], "label": "dog", '
'"mask": "bad", "confidence": 0.9}]'
)
xyxy, _, _, confidence, masks = from_google_gemini_2_5(
result=result, resolution_wh=(640, 480)
)
assert xyxy.shape == (2, 4)
assert confidence is not None
assert confidence.shape == (2,)
assert np.allclose(confidence, [0.8, 0.9])
assert masks is not None
assert masks.shape == (2, 480, 640)
@pytest.mark.parametrize(
("parser", "result"),
[
pytest.param(
from_google_gemini_2_0, "[1, 2, 3]", id="gemini_2_0_non_dict_items"
),
pytest.param(from_google_gemini_2_0, "42", id="gemini_2_0_non_list"),
pytest.param(
from_google_gemini_2_5, "[1, 2, 3]", id="gemini_2_5_non_dict_items"
),
pytest.param(from_google_gemini_2_5, "42", id="gemini_2_5_non_list"),
pytest.param(from_qwen_2_5_vl, "[1, 2, 3]", id="qwen_2_5_non_dict_items"),
pytest.param(from_qwen_2_5_vl, "42", id="qwen_2_5_non_list"),
],
)
def test_vlm_parsers_degrade_on_malformed_json(parser, result):
"""Valid JSON of the wrong shape should yield empty results, not raise."""
kwargs: dict = {"resolution_wh": (640, 480)}
if parser is from_qwen_2_5_vl:
kwargs["input_wh"] = (640, 480)
xyxy = parser(result=result, **kwargs)[0]
assert xyxy.shape == (0, 4)