From 9837c17878f4dd48a4353a68cb9d4fbc9681c8bd Mon Sep 17 00:00:00 2001 From: Piotr Skalski Date: Tue, 21 Jul 2026 16:20:05 +0200 Subject: [PATCH] feat(vlm): add Gemini 3.5 Flash parsing support (#2449) Add VLM.GOOGLE_GEMINI_3_5 enum and from_google_gemini_3_5 connector reusing the 2.5 parser, wire it into Detections.from_vlm, and salvage valid entries from partially malformed Gemini JSON arrays. Includes tests and changelog. Co-authored-by: Cursor Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> --- docs/changelog.md | 2 + src/supervision/detection/core.py | 18 +++++++ src/supervision/detection/vlm.py | 82 ++++++++++++++++++++++++++--- tests/detection/test_vlm.py | 85 +++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+), 8 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index adcd1f99..b4ccbfff 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -18,6 +18,7 @@ date_modified: 2026-07-21 - `sv.mask_non_max_merge` now computes exact mask overlap at the original mask resolution and ignores the deprecated `mask_dimension` parameter. Code that relied on downscaled mask overlap should recalibrate thresholds; passing `mask_dimension` positionally now emits a deprecation warning, and the parameter is scheduled for removal in `0.33.0` ([#2400](https://github.com/roboflow/supervision/pull/2400)). ### Fixed +- `sv.Detections.from_vlm` with `sv.VLM.GOOGLE_GEMINI_2_0`, `sv.VLM.GOOGLE_GEMINI_2_5`, and `sv.VLM.GOOGLE_GEMINI_3_5` now salvages the valid entries from a partially malformed JSON array (e.g. a single object with a syntax error) instead of discarding the whole response. - Geometry-aware IoU dispatch now powers the deprecated `merge_inner_detections_objects`, so overlapping axis-aligned envelopes no longer merge oriented boxes whose true OBB IoU is below the threshold ([#2374](https://github.com/roboflow/supervision/pull/2374)). - `save_coco_annotations` (and therefore `DetectionDataset.as_coco`) now reads image sizes from file headers via lazy PIL instead of cv2-decoding every image, so labels-only COCO exports no longer decode any pixel data ([#2442](https://github.com/roboflow/supervision/pull/2442)). - Fixed [#2437](https://github.com/roboflow/supervision/pull/2437): `sv.F1Score` no longer emits a spurious `RuntimeWarning` when true positives, false positives, and false negatives are all zero (denominator 0); the score remains `0.0`. @@ -62,6 +63,7 @@ date_modified: 2026-07-21 - Fixed: dataset IO/export edge cases now avoid mutating caller-owned `Detections` during `DetectionDataset` construction, reject non-integer and out-of-range class ids with a clear `ValueError`, load COCO annotations that omit optional `iscrowd`/`area` fields, expose `DetectionDataset.from_coco(use_iscrowd=...)` without changing the existing positional `show_progress` argument, export mask pixel area to COCO when no stored area is present, ignore folder-structure root clutter and non-image files inside class folders, and accept PIL-readable YOLO images such as RGBA or palette PNGs. ### Added +- `sv.VLM.GOOGLE_GEMINI_3_5` — `sv.Detections.from_vlm` now parses Google Gemini 3.5 output (detection and segmentation), reusing the Gemini 2.5 JSON format (`box_2d` + `label`, optional `mask`/`confidence`). - `sv.get_video_frames_generator` now accepts `prefetch: int = 0` ([#2273](https://github.com/roboflow/supervision/pull/2273)). When `> 0`, frames are decoded on a background daemon thread and buffered in a bounded queue, overlapping I/O with consumer processing. Default `0` preserves the existing synchronous behaviour. - Added a cv2-free PyAV fallback for file-video capture, writing, frame seeking, metadata, and `process_video(preserve_audio=True)` audio remuxing. OpenCV remains diff --git a/src/supervision/detection/core.py b/src/supervision/detection/core.py index 75321ade..51eea2bf 100644 --- a/src/supervision/detection/core.py +++ b/src/supervision/detection/core.py @@ -67,6 +67,7 @@ from supervision.detection.vlm import ( from_florence_2, from_google_gemini_2_0, from_google_gemini_2_5, + from_google_gemini_3_5, from_moondream, from_paligemma, from_qwen_2_5_vl, @@ -1142,6 +1143,7 @@ class Detections: | Qwen3-VL | `QWEN_3_VL` | detection | `resolution_wh` | `classes` | | Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` | | Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` | + | Google Gemini 3.5 | `GOOGLE_GEMINI_3_5` | detection, segmentation | `resolution_wh` | `classes` | | Moondream | `MOONDREAM` | detection | `resolution_wh` | | | DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` | | Qwen3-VL | `QWEN_3_VL` | detection | `resolution_wh` | `classes` | @@ -1622,6 +1624,7 @@ class Detections: | Qwen3-VL | `QWEN_3_VL` | detection | `resolution_wh` | `classes` | | Google Gemini 2.0 | `GOOGLE_GEMINI_2_0` | detection | `resolution_wh` | `classes` | | Google Gemini 2.5 | `GOOGLE_GEMINI_2_5` | detection, segmentation | `resolution_wh` | `classes` | + | Google Gemini 3.5 | `GOOGLE_GEMINI_3_5` | detection, segmentation | `resolution_wh` | `classes` | | Moondream | `MOONDREAM` | detection | `resolution_wh` | | | DeepSeek-VL2 | `DEEPSEEK_VL_2` | detection | `resolution_wh` | `classes` | @@ -2127,6 +2130,21 @@ class Detections: data=data, ) + if vlm == VLM.GOOGLE_GEMINI_3_5: + if not isinstance(result, str): + raise ValueError( + f"Invalid VLM result type: {type(result)}. Must be str." + ) + gemini_result = from_google_gemini_3_5(result, **kwargs) + data = {CLASS_NAME_DATA_FIELD: gemini_result[2]} + return cls( + xyxy=gemini_result[0], + class_id=gemini_result[1], + mask=gemini_result[4], + confidence=gemini_result[3], + data=data, + ) + raise ValueError(f"Unsupported VLM value: {vlm}.") @classmethod diff --git a/src/supervision/detection/vlm.py b/src/supervision/detection/vlm.py index 8cf05346..35edce07 100644 --- a/src/supervision/detection/vlm.py +++ b/src/supervision/detection/vlm.py @@ -82,6 +82,7 @@ class VLM(Enum): QWEN_3_VL: Qwen3-VL open vision-language model from Alibaba. GOOGLE_GEMINI_2_0: Google Gemini 2.0 vision-language model. GOOGLE_GEMINI_2_5: Google Gemini 2.5 vision-language model. + GOOGLE_GEMINI_3_5: Google Gemini 3.5 vision-language model. MOONDREAM: The Moondream vision-language model. """ @@ -92,6 +93,7 @@ class VLM(Enum): DEEPSEEK_VL_2 = "deepseek_vl_2" GOOGLE_GEMINI_2_0 = "gemini_2_0" GOOGLE_GEMINI_2_5 = "gemini_2_5" + GOOGLE_GEMINI_3_5 = "gemini_3_5" MOONDREAM = "moondream" @classmethod @@ -122,6 +124,7 @@ RESULT_TYPES: dict[VLM, type] = { VLM.DEEPSEEK_VL_2: str, VLM.GOOGLE_GEMINI_2_0: str, VLM.GOOGLE_GEMINI_2_5: str, + VLM.GOOGLE_GEMINI_3_5: str, VLM.MOONDREAM: dict, } @@ -133,6 +136,7 @@ REQUIRED_ARGUMENTS: dict[VLM, list[str]] = { VLM.DEEPSEEK_VL_2: ["resolution_wh"], VLM.GOOGLE_GEMINI_2_0: ["resolution_wh"], VLM.GOOGLE_GEMINI_2_5: ["resolution_wh"], + VLM.GOOGLE_GEMINI_3_5: ["resolution_wh"], VLM.MOONDREAM: ["resolution_wh"], } @@ -144,6 +148,7 @@ ALLOWED_ARGUMENTS: dict[VLM, list[str]] = { VLM.DEEPSEEK_VL_2: ["resolution_wh", "classes"], VLM.GOOGLE_GEMINI_2_0: ["resolution_wh", "classes"], VLM.GOOGLE_GEMINI_2_5: ["resolution_wh", "classes"], + VLM.GOOGLE_GEMINI_3_5: ["resolution_wh", "classes"], VLM.MOONDREAM: ["resolution_wh"], } @@ -602,6 +607,42 @@ def from_florence_2( raise RuntimeError(f"Unimplemented task: {task}") +def _recover_gemini_json_objects(text: str) -> list[Any]: + """ + Salvage individual JSON objects from a malformed Gemini JSON array. + + Scans for balanced `{...}` spans and parses each independently, keeping the + ones that decode into a `dict` and skipping the rest. This recovers the valid + entries from an array that a single `json.loads` would reject wholesale, such + as one whose objects contain a mid-array syntax error or a missing key. + + Args: + text: The (fence-stripped) response text that failed `json.loads`. + + Returns: + The list of successfully parsed objects, which may be empty. + """ + objects: list[Any] = [] + depth = 0 + start = None + for index, char in enumerate(text): + if char == "{": + if depth == 0: + start = index + depth += 1 + elif char == "}" and depth > 0: + depth -= 1 + if depth == 0 and start is not None: + try: + parsed = json.loads(text[start : index + 1]) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + objects.append(parsed) + start = None + return objects + + def from_google_gemini_2_0( result: str, resolution_wh: tuple[int, int], @@ -653,7 +694,7 @@ def from_google_gemini_2_0( try: data = json.loads(result) except json.JSONDecodeError: - return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str) + data = _recover_gemini_json_objects(result) if not isinstance(data, list): return np.empty((0, 4)), np.empty((0,), dtype=int), np.empty((0,), dtype=str) @@ -744,13 +785,7 @@ def from_google_gemini_2_5( try: data = json.loads(result) except json.JSONDecodeError: - return ( - np.empty((0, 4)), - np.array([], dtype=int), - np.array([], dtype=str), - np.array([], dtype=float), - None, - ) + data = _recover_gemini_json_objects(result) if not isinstance(data, list): return ( @@ -866,6 +901,37 @@ def from_google_gemini_2_5( ) +def from_google_gemini_3_5( + result: str, + resolution_wh: tuple[int, int], + classes: list[str] | None = None, +) -> tuple[ + npt.NDArray[Any], + npt.NDArray[Any] | None, + npt.NDArray[Any], + npt.NDArray[Any] | None, + npt.NDArray[Any] | None, +]: + """ + Parse and scale bounding boxes and masks from Google Gemini 3.5 style JSON output. + + Gemini 3.5 emits the same detection JSON as Gemini 2.5 (`box_2d` in + `[y_min, x_min, y_max, x_max]` normalized to 0-1000, plus `label` and optional + `mask`/`confidence`), so parsing delegates to `from_google_gemini_2_5`. + + Args: + result: String containing the JSON snippet enclosed by triple backticks. + resolution_wh: (output_width, output_height) to which we rescale the boxes. + classes: Optional list of valid class names. If provided, returned boxes/labels + are filtered to only those classes found here. + + Returns: + A tuple of `(xyxy, class_id, class_name, confidence, masks)` matching the + `from_google_gemini_2_5` return contract. + """ + return from_google_gemini_2_5(result, resolution_wh, classes) + + def from_moondream( result: dict[str, Any], resolution_wh: tuple[int, int], diff --git a/tests/detection/test_vlm.py b/tests/detection/test_vlm.py index 858ffdc7..1365bfa6 100644 --- a/tests/detection/test_vlm.py +++ b/tests/detection/test_vlm.py @@ -12,6 +12,7 @@ from supervision.detection.vlm import ( from_florence_2, from_google_gemini_2_0, from_google_gemini_2_5, + from_google_gemini_3_5, from_moondream, from_paligemma, from_qwen_2_5_vl, @@ -1437,6 +1438,7 @@ def test_from_vlm_unsupported_future_enum_raises( DEEPSEEK_VL_2 = object() GOOGLE_GEMINI_2_0 = object() GOOGLE_GEMINI_2_5 = object() + GOOGLE_GEMINI_3_5 = object() MOONDREAM = object() FUTURE = object() @@ -1462,6 +1464,10 @@ def test_from_vlm_unsupported_future_enum_raises( from_google_gemini_2_5, "[1, 2, 3]", id="gemini_2_5_non_dict_items" ), pytest.param(from_google_gemini_2_5, "42", id="gemini_2_5_non_list"), + pytest.param( + from_google_gemini_3_5, "[1, 2, 3]", id="gemini_3_5_non_dict_items" + ), + pytest.param(from_google_gemini_3_5, "42", id="gemini_3_5_non_list"), pytest.param(from_qwen_2_5_vl, "[1, 2, 3]", id="qwen_2_5_non_dict_items"), pytest.param(from_qwen_2_5_vl, "42", id="qwen_2_5_non_list"), ], @@ -1475,3 +1481,82 @@ def test_vlm_parsers_degrade_on_malformed_json(parser, result): xyxy = parser(result=result, **kwargs)[0] assert xyxy.shape == (0, 4) + + +def test_from_google_gemini_2_5_recovers_malformed_array(): + """A single broken entry must not discard the whole array; recover the rest.""" + result = ( + "```json\n" + "[\n" + ' {"box_2d": [10, 20, 110, 120], "label": "cat"},\n' + ' {"box_2d": [50, 100, 150, 200], "person"},\n' + ' {"box_2d": [30, 40, 130, 140], "label": "dog"}\n' + "]\n" + "```" + ) + + xyxy, _, class_name, _, _ = from_google_gemini_2_5( + result=result, resolution_wh=(640, 480) + ) + + assert xyxy.shape == (2, 4) + assert list(class_name) == ["cat", "dog"] + + +def test_from_google_gemini_2_0_recovers_malformed_array(): + """The 2.0 parser must also salvage valid entries around a broken one.""" + result = ( + "```json\n" + "[\n" + ' {"box_2d": [10, 20, 110, 120], "label": "cat"},\n' + ' {"box_2d": [50, 100, 150, 200], "person"},\n' + ' {"box_2d": [30, 40, 130, 140], "label": "dog"}\n' + "]\n" + "```" + ) + + xyxy, _, class_name = from_google_gemini_2_0( + result=result, resolution_wh=(640, 480) + ) + + assert xyxy.shape == (2, 4) + assert list(class_name) == ["cat", "dog"] + + +def test_from_google_gemini_3_5_parses_detections(): + """The 3.5 parser reuses the 2.5 format and returns boxes rescaled to resolution.""" + result = ( + "```json\n" + "[\n" + ' {"box_2d": [10, 20, 110, 120], "label": "cat"},\n' + ' {"box_2d": [50, 100, 150, 200], "label": "dog"}\n' + "]\n" + "```" + ) + + xyxy, _, class_name, _, _ = from_google_gemini_3_5( + result=result, resolution_wh=(640, 480) + ) + + assert xyxy.shape == (2, 4) + assert list(class_name) == ["cat", "dog"] + + +def test_from_google_gemini_3_5_recovers_malformed_array(): + """The 3.5 parser must salvage valid entries around a broken one, like 2.5.""" + result = ( + "```json\n" + "[\n" + ' {"box_2d": [10, 20, 110, 120], "label": "cat"},\n' + ' {"box_2d": [50, 100, 150, 200], "person"},\n' + ' {"box_2d": [30, 40, 130, 140], "label": "dog"}\n' + "]\n" + "```" + ) + + xyxy, _, class_name, _, _ = from_google_gemini_3_5( + result=result, resolution_wh=(640, 480) + ) + + assert xyxy.shape == (2, 4) + assert list(class_name) == ["cat", "dog"]