From 129537c7eb37ea0a9759d561b40a9887c3752887 Mon Sep 17 00:00:00 2001 From: chenjian Date: Mon, 18 Dec 2023 17:24:44 +0800 Subject: [PATCH] do not block ic by same req_id --- llm_ic/fastdeploy_ic/server/api.py | 7 ++++--- llm_ic/requirements.txt | 3 ++- llm_ic/setup.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/llm_ic/fastdeploy_ic/server/api.py b/llm_ic/fastdeploy_ic/server/api.py index ab0d48ae24..86ef27e0ea 100644 --- a/llm_ic/fastdeploy_ic/server/api.py +++ b/llm_ic/fastdeploy_ic/server/api.py @@ -41,12 +41,13 @@ async def ModelStreamInfer(self, request, context): # but we can not prevent two requests with the same req_id coming simultaneously. # To achieve this, we should add lock to query and insert query into redis, which will influence performance. # Currently, we assume different req_ids are confirmed by users. - if await data_manager.check_req_id_exist(model_id, req_id): - logger.info("ModelStreamInfer: req_id {}: has existed in other task".format(req_id)) - await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "ModelStreamInfer: req_id {}: has existed in other task".format(req_id)) + # if await data_manager.check_req_id_exist(model_id, req_id): + # logger.info("ModelStreamInfer: req_id {}: has existed in other task".format(req_id)) + # await context.abort(grpc.StatusCode.INVALID_ARGUMENT, "ModelStreamInfer: req_id {}: has existed in other task".format(req_id)) # 1. push request to redis await data_manager.add_req_id_to_map(model_id, req_id) await data_manager.enque_request(model_id, request) + logger.info("ModelStreamInfer: req_id {}: enqued request".format(req_id)) # 2. response stream results response_start_time = time.time() while True: diff --git a/llm_ic/requirements.txt b/llm_ic/requirements.txt index babf50a95c..d809ebeac7 100644 --- a/llm_ic/requirements.txt +++ b/llm_ic/requirements.txt @@ -1,3 +1,4 @@ aioredis colorlog -grpcio \ No newline at end of file +grpcio +protobuf \ No newline at end of file diff --git a/llm_ic/setup.py b/llm_ic/setup.py index dafda166a2..0b9b0ba0b5 100644 --- a/llm_ic/setup.py +++ b/llm_ic/setup.py @@ -15,7 +15,7 @@ import setuptools setuptools.setup( - name="fastdeploy-ic", + name="fastdeploy_ic", version="0.0.9", author="fastdeploy", author_email="fastdeploy@baidu.com", @@ -28,6 +28,6 @@ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ], - install_requires=["colorlog"], + install_requires=["colorlog", "aioredis", "grpcio", "protobuf"], extras_require={"client": ['grpcio', 'tritonclient']}, license='Apache 2.0')