From 6448fc7ea11d9e52ae45214f0ff42fcb82eb99a6 Mon Sep 17 00:00:00 2001
From: soumik12345 <19soumik.rakshit96@gmail.com>
Date: Thu, 10 Jul 2025 18:32:18 +0530
Subject: [PATCH] chore: address feedback on google gemini support
---
docs/detection/utils.md | 4 +-
supervision/__init__.py | 2 +-
supervision/detection/core.py | 100 +++++++++++++++++++++++----------
supervision/detection/utils.py | 6 +-
supervision/detection/vlm.py | 45 +++++++--------
5 files changed, 95 insertions(+), 62 deletions(-)
diff --git a/docs/detection/utils.md b/docs/detection/utils.md
index 94d50611..a8e9d696 100644
--- a/docs/detection/utils.md
+++ b/docs/detection/utils.md
@@ -102,10 +102,10 @@ status: new
:::supervision.detection.utils.xyxy_to_xywh
-:::supervision.detection.utils.normalized_xyxy_to_absolute_xyxy
+:::supervision.detection.utils.denormalize_boxes
diff --git a/supervision/__init__.py b/supervision/__init__.py
index c9ea3ce6..dba1d399 100644
--- a/supervision/__init__.py
+++ b/supervision/__init__.py
@@ -70,7 +70,7 @@ from supervision.detection.utils import (
mask_to_xyxy,
move_boxes,
move_masks,
- normalized_xyxy_to_absolute_xyxy,
+ denormalize_boxes,
oriented_box_iou_batch,
pad_boxes,
polygon_to_mask,
diff --git a/supervision/detection/core.py b/supervision/detection/core.py
index ec515f3e..ddc27838 100644
--- a/supervision/detection/core.py
+++ b/supervision/detection/core.py
@@ -847,39 +847,18 @@ class Detections:
Examples:
```python
- from google import genai
- from google.genai import types
import supervision as sv
- from PIL import Image
- IMAGE = Image.open()
- GENAI_CLIENT = genai.Client(api_key=)
+ gemini_response_text = \"\"\"```json
+ [
+ {"box_2d": [543, 40, 728, 200], "label": "Cat", "id": 1},
+ {"box_2d": [653, 352, 820, 522], "label": "Dog", "id": 2}
+ ]
+ ```\"\"\"
- system_instructions = '''
- Return bounding boxes as a JSON array with labels and ids. Never return masks or code fencing. Limit to 25 objects.
- If an object is present multiple times, name them according to their unique characteristic (colors, size, position, unique characteristics, etc..).
- '''
-
- safety_settings = [
- types.SafetySetting(
- category="HARM_CATEGORY_DANGEROUS_CONTENT",
- threshold="BLOCK_ONLY_HIGH",
- ),
- ]
-
- response = GENAI_CLIENT.models.generate_content(
- model="gemini-2.0-flash-exp",
- contents=[prompt, IMAGE],
- config = types.GenerateContentConfig(
- system_instruction=system_instructions,
- temperature=0.5,
- safety_settings=safety_settings,
- )
- )
-
- detections = sv.Detections.from_lmm(
- sv.LMM.GOOGLE_GEMINI_2_0,
- response.text,
+ detections = sv.Detections.from_vlm(
+ sv.VLM.GOOGLE_GEMINI_2_0,
+ gemini_response_text,
resolution_wh=(IMAGE.size[0], IMAGE.size[1]),
)
@@ -890,7 +869,6 @@ class Detections:
detections.data
# {'class_name': ['cat', 'dog']}
```
-
""" # noqa: E501 // docs
# filler logic mapping old from_lmm to new from_vlm
@@ -930,6 +908,66 @@ class Detections:
def from_vlm(
cls, vlm: Union[VLM, str], result: Union[str, dict], **kwargs: Any
) -> Detections:
+ """
+ Creates a Detections object from the given result string based on the specified
+ Vision Language Model (VLM).
+
+ Args:
+ vlm (Union[VLM, str]): The type of VLM (Large Multimodal Model) to use.
+ result (str): The result string containing the detection data.
+ **kwargs (Any): Additional keyword arguments required by the specified VLM.
+
+ Returns:
+ Detections: A new Detections object.
+
+ Raises:
+ ValueError: If the VLM is invalid, required arguments are missing, or
+ disallowed arguments are provided.
+ ValueError: If the specified VLM is not supported.
+
+ Examples:
+ ```python
+ import supervision as sv
+
+ paligemma_result = " cat"
+ detections = sv.Detections.from_vlm(
+ sv.VLM.PALIGEMMA,
+ paligemma_result,
+ resolution_wh=(1000, 1000),
+ classes=['cat', 'dog']
+ )
+ detections.xyxy
+ # array([[250., 250., 750., 750.]])
+
+ detections.class_id
+ # array([0])
+ ```
+
+ Examples:
+ ```python
+ import supervision as sv
+
+ gemini_response_text = \"\"\"```json
+ [
+ {"box_2d": [543, 40, 728, 200], "label": "Cat", "id": 1},
+ {"box_2d": [653, 352, 820, 522], "label": "Dog", "id": 2}
+ ]
+ ```\"\"\"
+
+ detections = sv.Detections.from_vlm(
+ sv.VLM.GOOGLE_GEMINI_2_0,
+ gemini_response_text,
+ resolution_wh=(IMAGE.size[0], IMAGE.size[1]),
+ )
+
+ detections.xyxy
+ # array([[250., 250., 750., 750.]])
+ detections.class_id
+ # array([0])
+ detections.data
+ # {'class_name': ['cat', 'dog']}
+ ```
+ """
vlm = validate_vlm_parameters(vlm, result, kwargs)
if vlm == VLM.PALIGEMMA:
diff --git a/supervision/detection/utils.py b/supervision/detection/utils.py
index da2858ec..8113ba43 100644
--- a/supervision/detection/utils.py
+++ b/supervision/detection/utils.py
@@ -447,7 +447,7 @@ def xyxy_to_xcycarh(xyxy: np.ndarray) -> np.ndarray:
return result.astype(float)
-def normalized_xyxy_to_absolute_xyxy(
+def denormalize_boxes(
normalized_xyxy: np.ndarray,
resolution_wh: Tuple[int, int],
normalization_factor: float = 1.0,
@@ -477,7 +477,7 @@ def normalized_xyxy_to_absolute_xyxy(
[0.3, 0.4, 0.7, 0.8]
])
resolution_wh = (100, 200)
- sv.normalized_xyxy_to_absolute_xyxy(normalized_xyxy, resolution_wh)
+ sv.denormalize_boxes(normalized_xyxy, resolution_wh)
# array([
# [ 10., 40., 50., 120.],
# [ 30., 80., 70., 160.]
@@ -487,7 +487,7 @@ def normalized_xyxy_to_absolute_xyxy(
[10., 20., 50., 60.],
[30., 40., 70., 80.]
])
- sv.normalized_xyxy_to_absolute_xyxy(normalized_xyxy, resolution_wh, max_value=100.0)
+ sv.denormalize_boxes(normalized_xyxy, resolution_wh, max_value=100.0)
# array([
# [ 10., 40., 50., 120.],
# [ 30., 80., 70., 160.]
diff --git a/supervision/detection/vlm.py b/supervision/detection/vlm.py
index b3342db7..15f28655 100644
--- a/supervision/detection/vlm.py
+++ b/supervision/detection/vlm.py
@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
import numpy as np
from supervision.detection.utils import (
- normalized_xyxy_to_absolute_xyxy,
+ denormalize_boxes,
polygon_to_mask,
polygon_to_xyxy,
)
@@ -22,11 +22,7 @@ class LMM(Enum):
FLORENCE_2 = "florence_2"
QWEN_2_5_VL = "qwen_2_5_vl"
GOOGLE_GEMINI_2_0 = "gemini_2_0"
- GOOGLE_GEMINI_2_0_FLASH_LITE = "gemini_2_0_flash_lite"
- GOOGLE_GEMINI_2_0_FLASH = "gemini_2_0_flash"
GOOGLE_GEMINI_2_5 = "gemini_2_5"
- GOOGLE_GEMINI_2_5_FLASH_PREVIEW = "gemini_2_5_flash_preview"
- GOOGLE_GEMINI_2_5_PRO_PREVIEW = "gemini_2_5_pro_preview"
class VLM(Enum):
@@ -34,11 +30,7 @@ class VLM(Enum):
FLORENCE_2 = "florence_2"
QWEN_2_5_VL = "qwen_2_5_vl"
GOOGLE_GEMINI_2_0 = "gemini_2_0"
- GOOGLE_GEMINI_2_0_FLASH_LITE = "gemini_2_0_flash_lite"
- GOOGLE_GEMINI_2_0_FLASH = "gemini_2_0_flash"
GOOGLE_GEMINI_2_5 = "gemini_2_5"
- GOOGLE_GEMINI_2_5_FLASH_PREVIEW = "gemini_2_5_flash_preview"
- GOOGLE_GEMINI_2_5_PRO_PREVIEW = "gemini_2_5_pro_preview"
RESULT_TYPES: Dict[VLM, type] = {
@@ -47,10 +39,6 @@ RESULT_TYPES: Dict[VLM, type] = {
VLM.QWEN_2_5_VL: str,
VLM.GOOGLE_GEMINI_2_0: str,
VLM.GOOGLE_GEMINI_2_5: str,
- VLM.GOOGLE_GEMINI_2_0_FLASH_LITE: str,
- VLM.GOOGLE_GEMINI_2_0_FLASH: str,
- VLM.GOOGLE_GEMINI_2_5_FLASH_PREVIEW: str,
- VLM.GOOGLE_GEMINI_2_5_PRO_PREVIEW: str,
}
REQUIRED_ARGUMENTS: Dict[VLM, List[str]] = {
@@ -59,10 +47,6 @@ REQUIRED_ARGUMENTS: Dict[VLM, List[str]] = {
VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh"],
VLM.GOOGLE_GEMINI_2_0: ["resolution_wh"],
VLM.GOOGLE_GEMINI_2_5: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_0_FLASH_LITE: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_0_FLASH: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_5_FLASH_PREVIEW: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_5_PRO_PREVIEW: ["resolution_wh"],
}
ALLOWED_ARGUMENTS: Dict[VLM, List[str]] = {
@@ -71,10 +55,6 @@ ALLOWED_ARGUMENTS: Dict[VLM, List[str]] = {
VLM.QWEN_2_5_VL: ["input_wh", "resolution_wh", "classes"],
VLM.GOOGLE_GEMINI_2_0: ["resolution_wh"],
VLM.GOOGLE_GEMINI_2_5: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_0_FLASH_LITE: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_0_FLASH: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_5_FLASH_PREVIEW: ["resolution_wh"],
- VLM.GOOGLE_GEMINI_2_5_PRO_PREVIEW: ["resolution_wh"],
}
SUPPORTED_TASKS_FLORENCE_2 = [
@@ -357,9 +337,24 @@ def from_google_gemini(
resolution_wh: Tuple[int, int],
) -> Tuple[np.ndarray, np.ndarray]:
"""
- Parse and scale bounding boxes from Google Gemini style JSON output.
- https://aistudio.google.com/
- https://ai.google.dev/gemini-api/docs/vision?lang=python
+ Parse and scale bounding boxes from Google Gemini style
+ [JSON output](https://ai.google.dev/gemini-api/docs/vision?lang=python).
+
+ The JSON is expected to be enclosed in triple backticks with the format:
+ ```json
+ [
+ {"box_2d": [x1, y1, x2, y2], "label": "some class name"},
+ ...
+ ]
+ ```
+
+ For example:
+ ```json
+ [
+ {"box_2d": [10, 20, 110, 120], "label": "cat"},
+ {"box_2d": [50, 100, 150, 200], "label": "dog"}
+ ]
+ ```
Args:
result: String containing the JSON snippet enclosed by triple backticks.
@@ -400,7 +395,7 @@ def from_google_gemini(
box = item["box_2d"]
# Gemini bbox order is [y_min, x_min, y_max, x_max]
xyxy.append(
- normalized_xyxy_to_absolute_xyxy(
+ denormalize_boxes(
np.array([box[1], box[0], box[3], box[2]]).astype(np.float64),
resolution_wh=(w, h),
normalization_factor=1000,