Merge pull request #268 from roboflow/feature/yolo-world

YOLO-World
roboflow · Feb 16, 2024 · 4ac428b · 4ac428b
2 parents 2ba388f + d35338c
commit 4ac428b
Show file tree

Hide file tree

Showing 32 changed files with 580 additions and 23 deletions.
diff --git a/.github/workflows/test.jetson_4.6.1.yml b/.github/workflows/test.jetson_4.6.1.yml
@@ -37,7 +37,7 @@ jobs:
           PORT=9101 INFERENCE_SERVER_REPO=roboflow-inference-server-jetson-4.6.1 make start_test_docker_jetson
       - name: 🧪 Regression Tests - Jetson 4.6.1
         run: |
-          SKIP_SPEED_TEST=true SKIP_DOCTR_TEST=true SKIP_CLIP_TEST=true SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_GROUNDING_DINO_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
+          SKIP_YOLO_WORLD_TEST=true SKIP_SPEED_TEST=true SKIP_DOCTR_TEST=true SKIP_CLIP_TEST=true SKIP_VISUALISATION_TESTS=true MAX_WAIT=300 SKIP_GROUNDING_DINO_TEST=true SKIP_YOLOV8_TEST=true SKIP_GAZE_TEST=true FUNCTIONAL=true PORT=9101 API_KEY=${{ secrets.API_KEY }} asl_instance_segmentation_API_KEY=${{ secrets.ASL_INSTANCE_SEGMENTATION_API_KEY }} asl_poly_instance_seg_API_KEY=${{ secrets.ASL_POLY_INSTANCE_SEG_API_KEY }} bccd_favz3_API_KEY=${{ secrets.BCCD_FAVZ3_API_KEY }} bccd_i4nym_API_KEY=${{ secrets.BCCD_I4NYM_API_KEY }} cats_and_dogs_smnpl_API_KEY=${{ secrets.CATS_AND_DOGS_SMNPL_API_KEY }} coins_xaz9i_API_KEY=${{ secrets.COINS_XAZ9I_API_KEY }} melee_API_KEY=${{ secrets.MELEE_API_KEY }} yolonas_test_API_KEY=${{ secrets.YOLONAS_TEST_API_KEY }} python3 -m pytest tests/inference/integration_tests/
       - name: 🧹 Cleanup Test Docker - Jetson 4.6.1
         run: make stop_test_docker
         if: success() || failure()
diff --git a/.release/pypi/inference.core.setup.py b/.release/pypi/inference.core.setup.py
@@ -60,10 +60,14 @@ def read_requirements(path):
         "cpu": read_requirements("requirements/requirements.cpu.txt"),
         "gaze": read_requirements("requirements/requirements.gaze.txt"),
         "gpu": read_requirements("requirements/requirements.gpu.txt"),
+        "grounding-dino": read_requirements(
+            "requirements/requirements.groundingdino.txt"
+        ),
         "hosted": read_requirements("requirements/requirements.hosted.txt"),
         "http": read_requirements("requirements/requirements.http.txt"),
         "sam": read_requirements("requirements/requirements.sam.txt"),
         "waf": read_requirements("requirements/requirements.waf.txt"),
+        "yolo-world": read_requirements("requirements/requirements.yolo_world.txt"),
     },
     classifiers=[
         "Programming Language :: Python :: 3",

diff --git a/.release/pypi/inference.cpu.setup.py b/.release/pypi/inference.cpu.setup.py
@@ -59,10 +59,14 @@ def read_requirements(path):
     extras_require={
         "clip": read_requirements("requirements/requirements.clip.txt"),
         "gaze": read_requirements("requirements/requirements.gaze.txt"),
+        "grounding-dino": read_requirements(
+            "requirements/requirements.groundingdino.txt"
+        ),
         "hosted": read_requirements("requirements/requirements.hosted.txt"),
         "http": read_requirements("requirements/requirements.http.txt"),
         "sam": read_requirements("requirements/requirements.sam.txt"),
         "waf": read_requirements("requirements/requirements.waf.txt"),
+        "yolo-world": read_requirements("requirements/requirements.yolo_world.txt"),
     },
     classifiers=[
         "Programming Language :: Python :: 3",

diff --git a/.release/pypi/inference.gpu.setup.py b/.release/pypi/inference.gpu.setup.py
@@ -59,10 +59,14 @@ def read_requirements(path):
     extras_require={
         "clip": read_requirements("requirements/requirements.clip.txt"),
         "gaze": read_requirements("requirements/requirements.gaze.txt"),
+        "grounding-dino": read_requirements(
+            "requirements/requirements.groundingdino.txt"
+        ),
         "hosted": read_requirements("requirements/requirements.hosted.txt"),
         "http": read_requirements("requirements/requirements.http.txt"),
         "sam": read_requirements("requirements/requirements.sam.txt"),
         "waf": read_requirements("requirements/requirements.waf.txt"),
+        "yolo-world": read_requirements("requirements/requirements.yolo_world.txt"),
     },
     classifiers=[
         "Programming Language :: Python :: 3",

diff --git a/.release/pypi/inference.setup.py b/.release/pypi/inference.setup.py
@@ -59,10 +59,14 @@ def read_requirements(path):
     extras_require={
         "clip": read_requirements("requirements/requirements.clip.txt"),
         "gaze": read_requirements("requirements/requirements.gaze.txt"),
+        "grounding-dino": read_requirements(
+            "requirements/requirements.groundingdino.txt"
+        ),
         "hosted": read_requirements("requirements/requirements.hosted.txt"),
         "http": read_requirements("requirements/requirements.http.txt"),
         "sam": read_requirements("requirements/requirements.sam.txt"),
         "waf": read_requirements("requirements/requirements.waf.txt"),
+        "yolo-world": read_requirements("requirements/requirements.yolo_world.txt"),
     },
     classifiers=[
         "Programming Language :: Python :: 3",

diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu b/docker/dockerfiles/Dockerfile.onnx.cpu
@@ -21,6 +21,7 @@ COPY requirements/requirements.sam.txt \
     requirements/requirements.gaze.txt \
     requirements/requirements.doctr.txt \
     requirements/requirements.groundingdino.txt \
+    requirements/requirements.yolo_world.txt \
     requirements/_requirements.txt \
     ./
 
@@ -34,6 +35,7 @@ RUN pip3 install --upgrade pip  && pip3 install \
     -r requirements.gaze.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    -r requirements.yolo_world.txt \
     jupyterlab \
     wheel>=0.38.0 \
     setuptools>=65.5.1 \

diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu b/docker/dockerfiles/Dockerfile.onnx.gpu
@@ -21,6 +21,7 @@ COPY requirements/requirements.sam.txt \
     requirements/requirements.doctr.txt \
     requirements/requirements.groundingdino.txt \
     requirements/requirements.cogvlm.txt \
+    requirements/requirements.yolo_world.txt \
     requirements/_requirements.txt \
     ./
 
@@ -35,6 +36,7 @@ RUN pip3 install --upgrade pip  && pip3 install \
     -r requirements.groundingdino.txt \
     -r requirements.doctr.txt \
     -r requirements.cogvlm.txt \
+    -r requirements.yolo_world.txt \
     jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip

diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0 b/docker/dockerfiles/Dockerfile.onnx.jetson.4.5.0
@@ -27,6 +27,7 @@ COPY requirements/requirements.clip.txt \
     requirements/requirements.http.txt \
     requirements/requirements.doctr.txt \
     requirements/requirements.groundingdino.txt \
+    requirements/requirements.yolo_world.txt \
     requirements/_requirements.txt \ 
     ./
 
@@ -38,6 +39,7 @@ RUN python3.8 -m pip install --upgrade pip  && python3.8 -m pip install \
     -r requirements.http.txt \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
+    -r requirements.yolo_world.txt \
     jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip

diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1 b/docker/dockerfiles/Dockerfile.onnx.jetson.4.6.1
@@ -43,6 +43,7 @@ COPY requirements/requirements.clip.txt \
     requirements/requirements.doctr.txt \
     requirements/requirements.groundingdino.txt \
     requirements/requirements.sdk.http.txt \
+    requirements/requirements.yolo_world.txt \
     requirements/_requirements.txt \
     ./
 
@@ -55,6 +56,7 @@ RUN python3.9 -m pip install --upgrade pip  && python3.9 -m pip install \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
     -r requirements.sdk.http.txt \
+    -r requirements.yolo_world.txt \
     jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip

diff --git a/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1 b/docker/dockerfiles/Dockerfile.onnx.jetson.5.1.1
@@ -27,6 +27,7 @@ COPY requirements/requirements.clip.txt \
     requirements/requirements.doctr.txt \
     requirements/requirements.groundingdino.txt \
     requirements/requirements.sdk.http.txt \
+    requirements/requirements.yolo_world.txt \
     requirements/_requirements.txt \
     ./
 
@@ -40,6 +41,7 @@ RUN python3.9 -m pip install --upgrade pip  && python3.9 -m pip install \
     -r requirements.doctr.txt \
     -r requirements.groundingdino.txt \
     -r requirements.sdk.http.txt \
+    -r requirements.yolo_world.txt \
     jupyterlab \
     --upgrade \
     && rm -rf ~/.cache/pip

diff --git a/docker/dockerfiles/Dockerfile.onnx.lambda b/docker/dockerfiles/Dockerfile.onnx.lambda
@@ -15,6 +15,7 @@ RUN yum update -y && yum install -y \
     gcc \
     mesa-libGL \
     pango \
+    wget \
     && yum clean all
 
 COPY requirements/requirements.clip.txt \
@@ -25,6 +26,7 @@ COPY requirements/requirements.clip.txt \
     requirements/requirements.groundingdino.txt \
     requirements/_requirements.txt \
     requirements/requirements.sdk.http.txt \
+    requirements/requirements.yolo_world.txt \
     ./
 
 RUN /var/lang/bin/python3.10 -m pip install --upgrade pip && rm -rf ~/.cache/pip
@@ -39,11 +41,15 @@ RUN pip3 install \
     -r requirements.groundingdino.txt \
     -r requirements.doctr.txt \
     -r requirements.sdk.http.txt \
+    -r requirements.yolo_world.txt \
     mangum \
     --upgrade \
     --target "${LAMBDA_TASK_ROOT}" \
     && rm -rf ~/.cache/pip
 
+WORKDIR /root/.cache/clip
+RUN wget https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt
+
 COPY inference ${LAMBDA_TASK_ROOT}/inference
 COPY inference_sdk ${LAMBDA_TASK_ROOT}/inference_sdk
 COPY docker/config/lambda.py ${LAMBDA_TASK_ROOT}/lambda.py

diff --git a/inference/core/cache/base.py b/inference/core/cache/base.py
@@ -102,3 +102,29 @@ def lock(self, key: str, expire: float = None) -> Any:
         finally:
             logger.debug(f"Releasing lock at cache key: {key}")
             l.release()
+
+    def set_numpy(self, key: str, value: Any, expire: float = None):
+        """
+        Caches a numpy array.
+
+        Args:
+            key (str): The key to store the value.
+            value (Any): The value to store.
+            expire (float, optional): The time, in seconds, after which the key will expire. Defaults to None.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError()
+
+    def get_numpy(self, key: str) -> Any:
+        """
+        Retrieves a numpy array from the cache.
+
+        Args:
+            key (str): The key of the value to retrieve.
+
+        Raises:
+            NotImplementedError: This method must be implemented by subclasses.
+        """
+        raise NotImplementedError()
diff --git a/inference/core/cache/memory.py b/inference/core/cache/memory.py
@@ -164,3 +164,9 @@ def acquire_lock(self, key: str, expire=None) -> Any:
         # refresh the lock
         self.set(key, lock, expire=expire)
         return lock
+
+    def set_numpy(self, key: str, value: Any, expire: float = None):
+        return self.set(key, value, expire=expire)
+
+    def get_numpy(self, key: str):
+        return self.get(key)
diff --git a/inference/core/cache/redis.py b/inference/core/cache/redis.py
@@ -1,6 +1,7 @@
 import asyncio
 import inspect
 import json
+import pickle
 import threading
 import time
 from contextlib import asynccontextmanager
@@ -86,7 +87,10 @@ def get(self, key: str):
         """
         item = self.client.get(key)
         if item is not None:
-            return json.loads(item)
+            try:
+                return json.loads(item)
+            except TypeError:
+                return item
 
     def set(self, key: str, value: str, expire: float = None):
         """
@@ -97,7 +101,9 @@ def set(self, key: str, value: str, expire: float = None):
             value (str): The value to store.
             expire (float, optional): The time, in seconds, after which the key will expire. Defaults to None.
         """
-        self.client.set(key, json.dumps(value), ex=expire)
+        if not isinstance(value, bytes):
+            value = json.dumps(value)
+        self.client.set(key, value, ex=expire)
 
     def zadd(self, key: str, value: Any, score: float, expire: float = None):
         """
@@ -177,3 +183,14 @@ def acquire_lock(self, key: str, expire=None) -> Any:
         if expire is not None:
             l.extend(expire)
         return l
+
+    def set_numpy(self, key: str, value: Any, expire: float = None):
+        serialized_value = pickle.dumps(value)
+        self.set(key, serialized_value, expire=expire)
+
+    def get_numpy(self, key: str) -> Any:
+        serialized_value = self.get(key)
+        if serialized_value is not None:
+            return pickle.loads(serialized_value)
+        else:
+            return None
diff --git a/inference/core/entities/requests/dynamic_class_base.py b/inference/core/entities/requests/dynamic_class_base.py
@@ -0,0 +1,19 @@
+from typing import List, Optional
+
+from pydantic import Field
+
+from inference.core.entities.requests.inference import CVInferenceRequest
+
+
+class DynamicClassBaseInferenceRequest(CVInferenceRequest):
+    """Request for zero-shot object detection models (with dynamic class lists).
+
+    Attributes:
+        text (List[str]): A list of strings.
+    """
+
+    model_id: Optional[str] = Field(None)
+    text: List[str] = Field(
+        examples=[["person", "dog", "cat"]],
+        description="A list of strings",
+    )
diff --git a/inference/core/entities/requests/groundingdino.py b/inference/core/entities/requests/groundingdino.py
@@ -1,20 +1,15 @@
 from typing import List, Optional
 
-from pydantic import Field
+from inference.core.entities.requests.dynamic_class_base import (
+    DynamicClassBaseInferenceRequest,
+)
 
-from inference.core.entities.requests.inference import CVInferenceRequest
 
-
-class GroundingDINOInferenceRequest(CVInferenceRequest):
+class GroundingDINOInferenceRequest(DynamicClassBaseInferenceRequest):
     """Request for Grounding DINO zero-shot predictions.
 
     Attributes:
         text (List[str]): A list of strings.
     """
 
-    model_id: Optional[str] = Field(None)
-    text: List[str] = Field(
-        examples=[["person", "dog", "cat"]],
-        description="A list of strings",
-    )
     grounding_dino_version_id: Optional[str] = "default"
diff --git a/inference/core/entities/requests/yolo_world.py b/inference/core/entities/requests/yolo_world.py
@@ -0,0 +1,17 @@
+from typing import List, Optional
+
+from inference.core.entities.requests.dynamic_class_base import (
+    DynamicClassBaseInferenceRequest,
+)
+from inference.core.models.defaults import DEFAULT_CONFIDENCE
+
+
+class YOLOWorldInferenceRequest(DynamicClassBaseInferenceRequest):
+    """Request for Grounding DINO zero-shot predictions.
+
+    Attributes:
+        text (List[str]): A list of strings.
+    """
+
+    yolo_world_version_id: Optional[str] = "l"
+    confidence: Optional[float] = DEFAULT_CONFIDENCE
diff --git a/inference/core/env.py b/inference/core/env.py
@@ -99,6 +99,11 @@
 # Flag to enable CogVLM core model, default is True
 CORE_MODEL_COGVLM_ENABLED = str2bool(os.getenv("CORE_MODEL_COGVLM_ENABLED", True))
 
+# Flag to enable YOLO-World core model, default is True
+CORE_MODEL_YOLO_WORLD_ENABLED = str2bool(
+    os.getenv("CORE_MODEL_YOLO_WORLD_ENABLED", True)
+)
+
 # ID of host device, default is None
 DEVICE_ID = os.getenv("DEVICE_ID", None)