From 5939be609c8c0506de0f520496eb3ae871886027 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 16 Jan 2024 16:53:55 -0800 Subject: [PATCH] Implement LLM API route --- diana_services_api/app/__init__.py | 2 + diana_services_api/app/routers/llm.py | 58 +++++++++++++++++++++++ diana_services_api/mq_service_api.py | 16 ++++++- diana_services_api/schema/llm_requests.py | 54 +++++++++++++++++++++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 diana_services_api/app/routers/llm.py create mode 100644 diana_services_api/schema/llm_requests.py diff --git a/diana_services_api/app/__init__.py b/diana_services_api/app/__init__.py index 5bb95ef..8a83bbf 100644 --- a/diana_services_api/app/__init__.py +++ b/diana_services_api/app/__init__.py @@ -28,6 +28,7 @@ from diana_services_api.app.dependencies import client_manager, jwt_bearer, mq_connector from diana_services_api.app.routers.api_proxy import proxy_route +from diana_services_api.app.routers.llm import llm_route from diana_services_api.app.routers.mq_backend import mq_route from diana_services_api.app.routers.auth import auth_route from diana_services_api.version import __version__ @@ -41,5 +42,6 @@ def create_app(config: dict): app.include_router(auth_route) app.include_router(proxy_route) app.include_router(mq_route) + app.include_router(llm_route) return app diff --git a/diana_services_api/app/routers/llm.py b/diana_services_api/app/routers/llm.py new file mode 100644 index 0000000..9c5e575 --- /dev/null +++ b/diana_services_api/app/routers/llm.py @@ -0,0 +1,58 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from fastapi import APIRouter, Depends +from diana_services_api.schema.llm_requests import * +from diana_services_api.app.dependencies import jwt_bearer, mq_connector + + +llm_route = APIRouter(prefix="/llm", tags=["backend"], + dependencies=[Depends(jwt_bearer)]) + + +@llm_route.post("/chatgpt") +async def llm_ask_chatgpt(query: LLMRequest) -> LLMResponse: + return mq_connector.query_llm("chat_gpt", **dict(query)) + + +@llm_route.post("/fastchat") +async def llm_ask_fastchat(query: LLMRequest) -> LLMResponse: + return mq_connector.query_llm("fastchat", **dict(query)) + + +@llm_route.post("/gemini") +async def llm_ask_gemini(query: LLMRequest) -> LLMResponse: + return mq_connector.query_llm("gemini", **dict(query)) + + +@llm_route.post("/claude") +async def llm_ask_claude(query: LLMRequest) -> LLMResponse: + return mq_connector.query_llm("claude", **dict(query)) + + +@llm_route.post("/palm") +async def llm_ask_palm(query: LLMRequest) -> LLMResponse: + return mq_connector.query_llm("palm2", **dict(query)) diff --git a/diana_services_api/mq_service_api.py b/diana_services_api/mq_service_api.py index 84dbc20..224abfa 100644 --- a/diana_services_api/mq_service_api.py +++ b/diana_services_api/mq_service_api.py @@ -25,7 +25,8 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json -from typing import Optional, Dict, Any +from typing import Optional, Dict, Any, List +from uuid import uuid4 from fastapi import HTTPException @@ -41,6 +42,7 @@ class APIError(HTTPException): class MQServiceManager: def __init__(self, config: dict): self.mq_default_timeout = config.get('mq_default_timeout', 10) + self.mq_cliend_id = str(uuid4()) def _validate_api_proxy_response(self, response: dict): if response['status_code'] == 200: @@ -67,6 +69,18 @@ def query_api_proxy(self, service_name: str, query_params: dict, "neon_api_output", timeout) return self._validate_api_proxy_response(response) + def query_llm(self, llm_name: str, query: str, history: List[tuple]): + response = send_mq_request("/llm", {"query": query, + "history": history}, + f"{llm_name}_input", + response_queue=f"{llm_name}_" + f"{self.mq_cliend_id}") + response = response.get('response') or "" + history.append(("user", query)) + history.append(("llm", response)) + return {"response": response, + "history": history} + def send_email(self, recipient: str, subject: str, body: str, attachments: Optional[Dict[str, str]]): request_data = {"recipient": recipient, diff --git a/diana_services_api/schema/llm_requests.py b/diana_services_api/schema/llm_requests.py new file mode 100644 index 0000000..1861f4d --- /dev/null +++ b/diana_services_api/schema/llm_requests.py @@ -0,0 +1,54 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import List + +from pydantic import BaseModel + + +class LLMRequest(BaseModel): + query: str + history: List[tuple] = [] + model_config = { + "json_schema_extra": { + "examples": [{ + "query": "I am well, how about you?", + "history": [("user", "hello"), + ("llm", "Hi, how can I help you today?")]}]}} + + +class LLMResponse(BaseModel): + response: str + history: List[tuple] + model_config = { + "json_schema_extra": { + "examples": [{ + "query": "I am well, how about you?", + "history": [("user", "hello"), + ("llm", "Hi, how can I help you today?"), + ("user", "I am well, how about you?"), + ("llm", "As a large language model, I do not feel") + ]}]}}