feat(plugin): support ai-proxy-multi (#11986)

Author: shreemaan-abhishek

Makefile

@@ -374,8 +374,8 @@ install: runtime
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-proxy
 	$(ENV_INSTALL) apisix/plugins/ai-proxy/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ai-proxy
 
-	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-proxy/drivers
-	$(ENV_INSTALL) apisix/plugins/ai-proxy/drivers/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ai-proxy/drivers
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-drivers
+	$(ENV_INSTALL) apisix/plugins/ai-drivers/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ai-drivers
 
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ai-rag/embeddings
 	$(ENV_INSTALL) apisix/plugins/ai-rag/embeddings/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ai-rag/embeddings

apisix/cli/config.lua

@@ -223,6 +223,7 @@ local _M = {
     "workflow",
     "api-breaker",
     "ai-proxy",
+    "ai-proxy-multi",
     "limit-conn",
     "limit-count",
     "limit-req",

apisix/plugins/ai-drivers/deepseek.lua

@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+return require("apisix.plugins.ai-drivers.openai-compatible").new(
+    {
+        host = "api.deepseek.com",
+        path = "/chat/completions",
+        port = 443
+    }
+)

apisix/plugins/ai-proxy/drivers/openai.lua apisix/plugins/ai-drivers/openai-compatible.lua

@@ -16,46 +16,57 @@
 --
 local _M = {}
 
+local mt = {
+    __index = _M
+}
+
 local core = require("apisix.core")
 local http = require("resty.http")
 local url  = require("socket.url")
 
 local pairs = pairs
 local type  = type
+local setmetatable = setmetatable
+
+
+function _M.new(opts)
 
--- globals
-local DEFAULT_HOST = "api.openai.com"
-local DEFAULT_PORT = 443
-local DEFAULT_PATH = "/v1/chat/completions"
+    local self = {
+        host = opts.host,
+        port = opts.port,
+        path = opts.path,
+    }
+    return setmetatable(self, mt)
+end
 
 
-function _M.request(conf, request_table, ctx)
+function _M.request(self, conf, request_table, extra_opts)
     local httpc, err = http.new()
     if not httpc then
         return nil, "failed to create http client to send request to LLM server: " .. err
     end
     httpc:set_timeout(conf.timeout)
 
-    local endpoint = core.table.try_read_attr(conf, "override", "endpoint")
+    local endpoint = extra_opts.endpoint
     local parsed_url
     if endpoint then
         parsed_url = url.parse(endpoint)
     end
 
     local ok, err = httpc:connect({
         scheme = endpoint and parsed_url.scheme or "https",
-        host = endpoint and parsed_url.host or DEFAULT_HOST,
-        port = endpoint and parsed_url.port or DEFAULT_PORT,
+        host = endpoint and parsed_url.host or self.host,
+        port = endpoint and parsed_url.port or self.port,
         ssl_verify = conf.ssl_verify,
-        ssl_server_name = endpoint and parsed_url.host or DEFAULT_HOST,
+        ssl_server_name = endpoint and parsed_url.host or self.host,
         pool_size = conf.keepalive and conf.keepalive_pool,
     })
 
     if not ok then
         return nil, "failed to connect to LLM server: " .. err
     end
 
-    local query_params = conf.auth.query or {}
+    local query_params = extra_opts.query_params
 
     if type(parsed_url) == "table" and parsed_url.query and #parsed_url.query > 0 then
         local args_tab = core.string.decode_args(parsed_url.query)
@@ -64,9 +75,9 @@ function _M.request(conf, request_table, ctx)
         end
     end
 
-    local path = (endpoint and parsed_url.path or DEFAULT_PATH)
+    local path = (endpoint and parsed_url.path or self.path)
 
-    local headers = (conf.auth.header or {})
+    local headers = extra_opts.headers
     headers["Content-Type"] = "application/json"
     local params = {
         method = "POST",
@@ -77,13 +88,14 @@ function _M.request(conf, request_table, ctx)
         query = query_params
     }
 
-    if conf.model.options then
-        for opt, val in pairs(conf.model.options) do
+    if extra_opts.model_options then
+        for opt, val in pairs(extra_opts.model_options) do
             request_table[opt] = val
         end
     end
     params.body = core.json.encode(request_table)
 
+    httpc:set_timeout(conf.keepalive_timeout)
     local res, err = httpc:request(params)
     if not res then
         return nil, err

apisix/plugins/ai-drivers/openai.lua

@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+return require("apisix.plugins.ai-drivers.openai-compatible").new(
+    {
+        host = "api.openai.com",
+        path = "/v1/chat/completions",
+        port = 443
+    }
+)