diff --git a/docs/changelog.md b/docs/changelog.md index db2e8b87..79ab0697 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -15,6 +15,8 @@ date_modified: 2026-06-16 - Fixed [#2322](https://github.com/roboflow/supervision/pull/2322): COCO export now preserves all polygon parts for multi-component masks. Previously, only the first polygon was written when a non-crowd mask had disjoint segments; all parts are now included. +- Fixed [#2342](https://github.com/roboflow/supervision/pull/2342): `sv.Detections.from_vlm` with `sv.VLM.GOOGLE_GEMINI_2_0`, `sv.VLM.GOOGLE_GEMINI_2_5`, and `sv.VLM.QWEN_2_5_VL` no longer raises when the model returns valid JSON of the wrong shape (non-list top-level or non-dict elements). A non-string or malformed `"mask"` value in Gemini 2.5 output no longer triggers `AttributeError`; invalid base64 or non-PNG mask data falls back to an empty mask, keeping `xyxy`, `confidence`, and `masks` arrays aligned. + - Fixed [#2333](https://github.com/roboflow/supervision/pull/2333): [`sv.DetectionsSmoother`](https://supervision.roboflow.com/latest/detection/tools/smoother/#supervision.detection.tools.smoother.DetectionsSmoother) no longer raises when smoothing detections without `confidence`. Confidence is now averaged over the frames that carry it; when tracks in the same frame disagree on confidence presence, `confidence` is set to `None` for all smoothed detections. - Fixed [#2341](https://github.com/roboflow/supervision/pull/2341): `sv.DetectionDataset.as_pascal_voc` no longer mutates the source `Detections.xyxy` by the 1-index offset on every call. Previously, repeated exports accumulated a `+1` shift in the caller's bounding boxes. diff --git a/src/supervision/detection/vlm.py b/src/supervision/detection/vlm.py index 0b0bc830..a4d1fb10 100644 --- a/src/supervision/detection/vlm.py +++ b/src/supervision/detection/vlm.py @@ -214,7 +214,7 @@ def validate_vlm_parameters(vlm: VLM | str, result: Any, kwargs: dict[str, Any]) def from_paligemma( result: str, resolution_wh: tuple[int, int], classes: list[str] | None = None -) -> tuple[npt.NDArray[Any], npt.NDArray[Any], npt.NDArray[Any]]: +) -> tuple[npt.NDArray[Any], npt.NDArray[Any] | None, npt.NDArray[Any]]: """ Parse bounding boxes from paligemma-formatted text, scale them to the specified resolution, and optionally filter by classes. @@ -238,23 +238,23 @@ def from_paligemma( r"(?) ([\w\s\-]+)" ) matches = pattern.findall(result) - matches = np.array(matches) if matches else np.empty((0, 5)) + matches_arr: npt.NDArray[Any] = np.array(matches) if matches else np.empty((0, 5)) - if matches.shape[0] == 0: + if matches_arr.shape[0] == 0: return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty(0, dtype=str) - xyxy, class_name = matches[:, [1, 0, 3, 2]], matches[:, 4] - xyxy = xyxy.astype(int) / 1024 * np.array([w, h, w, h]) - class_name = np.char.strip(class_name.astype(str)) - class_id = None + xyxy_arr = np.array(matches_arr[:, [1, 0, 3, 2]], dtype=float) + xyxy_arr = xyxy_arr.astype(int) / 1024 * np.array([w, h, w, h]) + class_name = np.char.strip(matches_arr[:, 4].astype(str)) + class_id: npt.NDArray[Any] | None = None if classes is not None: mask = np.array([name in classes for name in class_name], dtype=bool) - xyxy = xyxy[mask] + xyxy_arr = xyxy_arr[mask] class_name = class_name[mask] class_id = np.array([classes.index(name) for name in class_name]) - return xyxy, class_id, class_name + return xyxy_arr, class_id, class_name def recover_truncated_qwen_2_5_vl_response(text: str) -> Any | None: @@ -369,7 +369,7 @@ def from_qwen_2_5_vl( labels_list = [] for item in data: - if "bbox_2d" not in item or "label" not in item: + if not isinstance(item, dict) or "bbox_2d" not in item or "label" not in item: continue boxes_list.append(item["bbox_2d"]) labels_list.append(item["label"]) @@ -460,12 +460,13 @@ def from_deepseek_vl_2( f"and det tags ({len(detection_segments)}) in the result must be equal." ) - xyxy, class_name_list = [], [] + xyxy_list: list[list[float]] = [] + class_name_list: list[str] = [] for label, detection_blob in zip(label_segments, detection_segments): current_class_name = label.strip() for box in re.findall(r"\[(.*?)\]", detection_blob): x1, y1, x2, y2 = map(float, box.strip("[]").split(",")) - xyxy.append( + xyxy_list.append( [ (x1 / 999 * width), (y1 / 999 * height), @@ -475,7 +476,7 @@ def from_deepseek_vl_2( ) class_name_list.append(current_class_name) - xyxy = np.array(xyxy, dtype=np.float32) + xyxy = np.array(xyxy_list, dtype=np.float32) class_name = np.array(class_name_list) if classes is not None: @@ -541,15 +542,15 @@ def from_florence_2( return xyxy, labels, None, xyxyxyxy if task in ["", ""]: - xyxy_list = [] - masks_list = [] + xyxy_list: list[npt.NDArray[Any]] = [] + masks_list: list[npt.NDArray[Any]] = [] for polygons_of_same_class in result["polygons"]: for polygon in polygons_of_same_class: polygon = np.reshape(polygon, (-1, 2)).astype(np.int32) mask = polygon_to_mask(polygon, resolution_wh).astype(bool) masks_list.append(mask) - xyxy = polygon_to_xyxy(polygon) - xyxy_list.append(xyxy) + xyxy_box = polygon_to_xyxy(polygon) + xyxy_list.append(xyxy_box) # per-class labels also provided, but they are ["", "", "", ...] # when we figure out how to set class names, we can do # zip(result["labels"], result["polygons"]) @@ -640,11 +641,14 @@ def from_google_gemini_2_0( except json.JSONDecodeError: return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str) + if not isinstance(data, list): + return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str) + labels = [] xyxy = [] for item in data: - if "box_2d" not in item or "label" not in item: + if not isinstance(item, dict) or "box_2d" not in item or "label" not in item: continue labels.append(item["label"]) box = item["box_2d"] @@ -734,13 +738,22 @@ def from_google_gemini_2_5( None, ) + if not isinstance(data, list): + return ( + np.empty((0, 4)), + np.array([], dtype=int), + np.array([], dtype=str), + np.array([], dtype=float), + None, + ) + boxes_list: list[Any] = [] labels_list: list[str] = [] confidence_list: list[float] | None = [] masks_list: list[npt.NDArray[Any]] | None = [] for item in data: - if "box_2d" not in item or "label" not in item: + if not isinstance(item, dict) or "box_2d" not in item or "label" not in item: continue labels_list.append(item["label"]) box = item["box_2d"] @@ -755,29 +768,39 @@ def from_google_gemini_2_5( if "mask" in item: if masks_list is not None: png_str = item["mask"] - if not png_str.startswith("data:image/png;base64,"): + if not isinstance(png_str, str) or not png_str.startswith( + "data:image/png;base64," + ): + # Malformed mask: keep an empty mask but still fall through to + # the confidence handling below, so the per-item arrays stay + # aligned (a `continue` here desynced confidence vs boxes). masks_list.append(np.zeros((h, w), dtype=bool)) - continue - - png_str = png_str.removeprefix("data:image/png;base64,") - png_str = base64.b64decode(png_str) - mask_img = Image.open(io.BytesIO(png_str)) - - y_min, y_max = int(absolute_bbox[1]), int(absolute_bbox[3]) - x_min, x_max = int(absolute_bbox[0]), int(absolute_bbox[2]) - - bbox_height = y_max - y_min - bbox_width = x_max - x_min - - if bbox_height > 0 and bbox_width > 0: - mask_img = mask_img.resize( - (bbox_width, bbox_height), resample=Image.Resampling.BILINEAR - ) - np_mask: npt.NDArray[np.bool_] = np.zeros((h, w), dtype=bool) - np_mask[y_min:y_max, x_min:x_max] = np.array(mask_img) > 0 - masks_list.append(np_mask) else: - masks_list.append(np.zeros((h, w), dtype=bool)) + png_str = png_str.removeprefix("data:image/png;base64,") + try: + png_bytes = base64.b64decode(png_str) + mask_img = Image.open(io.BytesIO(png_bytes)).convert("L") + except Exception: + masks_list.append(np.zeros((h, w), dtype=bool)) + else: + y_min, y_max = int(absolute_bbox[1]), int(absolute_bbox[3]) + x_min, x_max = int(absolute_bbox[0]), int(absolute_bbox[2]) + + bbox_height = y_max - y_min + bbox_width = x_max - x_min + + if bbox_height > 0 and bbox_width > 0: + mask_img = mask_img.resize( + (bbox_width, bbox_height), + resample=Image.Resampling.BILINEAR, + ) + np_mask: npt.NDArray[np.bool_] = np.zeros( + (h, w), dtype=bool + ) + np_mask[y_min:y_max, x_min:x_max] = np.array(mask_img) > 0 + masks_list.append(np_mask) + else: + masks_list.append(np.zeros((h, w), dtype=bool)) else: masks_list = None diff --git a/tests/detection/test_vlm.py b/tests/detection/test_vlm.py index b5d035b0..7b191d9d 100644 --- a/tests/detection/test_vlm.py +++ b/tests/detection/test_vlm.py @@ -1297,3 +1297,49 @@ def test_from_deepseek_vl_2( detections.data[CLASS_NAME_DATA_FIELD], expected_detections.data[CLASS_NAME_DATA_FIELD], ) + + +def test_from_google_gemini_2_5_malformed_mask_keeps_confidence_aligned(): + """A non-data-URI mask must not skip the item's confidence and desync arrays.""" + result = ( + '[{"box_2d": [10, 10, 100, 100], "label": "cat", "mask": "bad", ' + '"confidence": 0.8}, {"box_2d": [20, 20, 120, 120], "label": "dog", ' + '"mask": "bad", "confidence": 0.9}]' + ) + + xyxy, _, _, confidence, masks = from_google_gemini_2_5( + result=result, resolution_wh=(640, 480) + ) + + assert xyxy.shape == (2, 4) + assert confidence is not None + assert confidence.shape == (2,) + assert np.allclose(confidence, [0.8, 0.9]) + assert masks is not None + assert masks.shape == (2, 480, 640) + + +@pytest.mark.parametrize( + ("parser", "result"), + [ + pytest.param( + from_google_gemini_2_0, "[1, 2, 3]", id="gemini_2_0_non_dict_items" + ), + pytest.param(from_google_gemini_2_0, "42", id="gemini_2_0_non_list"), + pytest.param( + from_google_gemini_2_5, "[1, 2, 3]", id="gemini_2_5_non_dict_items" + ), + pytest.param(from_google_gemini_2_5, "42", id="gemini_2_5_non_list"), + pytest.param(from_qwen_2_5_vl, "[1, 2, 3]", id="qwen_2_5_non_dict_items"), + pytest.param(from_qwen_2_5_vl, "42", id="qwen_2_5_non_list"), + ], +) +def test_vlm_parsers_degrade_on_malformed_json(parser, result): + """Valid JSON of the wrong shape should yield empty results, not raise.""" + kwargs: dict = {"resolution_wh": (640, 480)} + if parser is from_qwen_2_5_vl: + kwargs["input_wh"] = (640, 480) + + xyxy = parser(result=result, **kwargs)[0] + + assert xyxy.shape == (0, 4)