[Feat] Handle case where the tokenizer is not initialised

Signed-off-by: Michele Gazzetti <michele.gazzetti1@ibm.com>

Author: mgazz

vllm/entrypoints/openai/protocol.py

@@ -1109,7 +1109,7 @@ class EmbeddingChatRequest(OpenAIBaseModel):
     model: Optional[str] = None
     messages: list[ChatCompletionMessageParam]
 
-    encoding_format: Literal["float", "base64"] = "float"
+    encoding_format: Literal["float", "base64", "tensors"] = "float"
     dimensions: Optional[int] = None
     user: Optional[str] = None
     truncate_prompt_tokens: Optional[Annotated[int, Field(ge=-1)]] = None

vllm/entrypoints/openai/serving_engine.py

@@ -807,6 +807,8 @@ async def _preprocess_chat(
                 messages=messages,
                 **_chat_template_kwargs,
             )
+        elif tokenizer is None:
+            request_prompt = "placeholder"
         else:
             request_prompt = apply_hf_chat_template(
                 tokenizer=tokenizer,
@@ -831,7 +833,17 @@ async def _preprocess_chat(
             request = tool_parser(tokenizer).adjust_request(  # type: ignore
                 request=request)
 
-        if isinstance(request_prompt, str):
+        if tokenizer is None:
+            prompt_inputs = {}
+            if "prompt_token_ids" not in request.additional_data:
+                raise Exception("Request must contain "
+                                "additional_data['prompt_token_ids'] "
+                                "when the tokenizer is not initialised")
+
+            prompt_inputs["prompt_token_ids"] = request.additional_data[
+                "prompt_token_ids"]
+
+        elif isinstance(request_prompt, str):
             prompt_inputs = await self._tokenize_prompt_input_async(
                 request,
                 tokenizer,

vllm/entrypoints/openai/serving_pooling.py

@@ -25,6 +25,7 @@
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
 from vllm.entrypoints.utils import _validate_truncation_size
 from vllm.logger import init_logger
+from vllm.multimodal.image import ImageEmbeddingMediaIO
 from vllm.outputs import PoolingOutput, PoolingRequestOutput
 from vllm.utils import merge_async_iterators
 
@@ -33,7 +34,7 @@
 
 def _get_data(
     output: PoolingOutput,
-    encoding_format: Literal["float", "base64"],
+    encoding_format: Literal["float", "base64", "tensors"],
 ) -> Union[list[float], str]:
     if encoding_format == "float":
         return output.data.tolist()
@@ -43,6 +44,9 @@ def _get_data(
         pt_float32 = output.data.to(dtype=torch.float32)
         pooling_bytes = np.array(pt_float32, dtype="float32").tobytes()
         return base64.b64encode(pooling_bytes).decode("utf-8")
+    elif encoding_format == "tensors":
+        tensor_encoding_io = ImageEmbeddingMediaIO()
+        tensor_encoding_io.encode_base64(output.data)
 
     assert_never(encoding_format)
 
@@ -99,7 +103,11 @@ async def create_pooling(
                 prompt_adapter_request,
             ) = self._maybe_get_adapters(request)
 
-            tokenizer = await self.engine_client.get_tokenizer(lora_request)
+            if not self.model_config.skip_tokenizer_init:
+                tokenizer = await self.engine_client.get_tokenizer(lora_request
+                                                                   )
+            else:
+                tokenizer = None
 
             if prompt_adapter_request is not None:
                 raise NotImplementedError("Prompt adapter is not supported "
@@ -205,7 +213,7 @@ def request_output_to_pooling_response(
         request_id: str,
         created_time: int,
         model_name: str,
-        encoding_format: Literal["float", "base64"],
+        encoding_format: Literal["float", "base64", "tensors"],
     ) -> PoolingResponse:
         items: list[PoolingResponseData] = []
         num_prompt_tokens = 0

vllm/model_executor/models/prithvi_geospatial_mae.py

@@ -45,7 +45,7 @@
 class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
 
     def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
-        return {"image": None}
+        return {"image": None,"tensors":2}
 
 
 class PrithviGeoSpatialMAEInputBuilder(
@@ -101,7 +101,11 @@ def apply(
         mm_kwargs = {}
 
         for k, v in mm_data.items():
-            mm_kwargs[k] = v
+            if isinstance(v,dict) and k == "tensors":
+                for tensor_name,tensor in v.items():
+                    mm_kwargs[tensor_name] = tensor 
+            else:        
+                mm_kwargs[k] = v
         mm_place_holders = {"image": [PlaceholderRange(offset=0, length=0)]}
 
         multimodal_kwargs_items = [

vllm/v1/engine/async_llm.py

@@ -103,10 +103,13 @@ def __init__(
         )
 
         # Tokenizer (+ ensure liveness if running in another process).
-        self.tokenizer = init_tokenizer_from_configs(
-            model_config=vllm_config.model_config,
-            scheduler_config=vllm_config.scheduler_config,
-            lora_config=vllm_config.lora_config)
+        if not self.vllm_config.model_config.skip_tokenizer_init:
+            self.tokenizer = init_tokenizer_from_configs(
+                model_config=vllm_config.model_config,
+                scheduler_config=vllm_config.scheduler_config,
+                lora_config=vllm_config.lora_config)
+        else:
+            self.tokenizer = None
 
         # Processor (converts Inputs --> EngineCoreRequests).
         self.processor = Processor(