From f6f7a411f7111c81dbcd56ea67c1a89e98458682 Mon Sep 17 00:00:00 2001 From: sangchengmeng <101796078+SangChengC@users.noreply.github.com> Date: Wed, 26 Feb 2025 19:23:43 +0800 Subject: [PATCH] Fix tokens2 (#748) --- lightllm/models/internvl/model.py | 3 +++ lightllm/server/api_http.py | 3 ++- lightllm/server/httpserver/manager.py | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lightllm/models/internvl/model.py b/lightllm/models/internvl/model.py index d32769886..3e0699bb9 100644 --- a/lightllm/models/internvl/model.py +++ b/lightllm/models/internvl/model.py @@ -47,6 +47,9 @@ def get_image_token_length(self, img: ImageItem): def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs): # TEXTTEXTTEXT --> TEXTTEXTTEXT image_tokens = IMG_START_TOKEN + IMG_END_TOKEN + if multimodal_params is None: + add_special_tokens = kwargs.get("add_special_tokens", True) + return self.tokenizer.encode(prompt, add_special_tokens=add_special_tokens) image_count = len(multimodal_params.images) prompt = prompt.replace(IMG_TOKEN, image_tokens, image_count) diff --git a/lightllm/server/api_http.py b/lightllm/server/api_http.py index 464bbd898..667ac8cb7 100755 --- a/lightllm/server/api_http.py +++ b/lightllm/server/api_http.py @@ -325,7 +325,8 @@ async def tokens(request: Request): try: request_dict = await request.json() prompt = request_dict.pop("text") - return JSONResponse({"ntokens": g_objs.httpserver_manager.tokens(prompt)}, status_code=200) + parameters = request_dict.pop("parameters") + return JSONResponse({"ntokens": g_objs.httpserver_manager.tokens(prompt, parameters)}, status_code=200) except Exception as e: return create_error_response(HTTPStatus.EXPECTATION_FAILED, f"error: {str(e)}") diff --git a/lightllm/server/httpserver/manager.py b/lightllm/server/httpserver/manager.py index 056c217d1..b38327e25 100644 --- a/lightllm/server/httpserver/manager.py +++ b/lightllm/server/httpserver/manager.py @@ -121,8 +121,9 @@ async def _release_multimodal_resources(self, multimodal_params: MultimodalParam img.token_num = None return - def tokens(self, prompt): - prompt_ids = self.tokenizer.encode(prompt) + def tokens(self, prompt, kwargs=None): + kwargs = {} if kwargs is None else kwargs + prompt_ids = self.tokenizer.encode(prompt, None, **kwargs) return len(prompt_ids) async def generate(