Merge branch 'master' into consumer-api-doc-rewording

.requirements

@@ -4,15 +4,15 @@ OPENRESTY=1.25.3.1
 LUAROCKS=3.11.0
 OPENSSL=3.2.1
 PCRE=10.43
-LIBEXPAT=2.5.0
+LIBEXPAT=2.6.2
 
 # Note: git repositories can be loaded from local path if path is set as value
 
-LUA_KONG_NGINX_MODULE=691ba795ced07364d491e8abbdf0c8c8d3778c14 # 0.10.0
+LUA_KONG_NGINX_MODULE=a8411f7cf4289049f0bd3e8e40088e7256389ed3 # 0.11.0
 LUA_RESTY_LMDB=7d2581cbe30cde18a8482d820c227ca0845c0ded # 1.4.2
 LUA_RESTY_EVENTS=8448a92cec36ac04ea522e78f6496ba03c9b1fd8 # 0.2.0
 LUA_RESTY_WEBSOCKET=60eafc3d7153bceb16e6327074e0afc3d94b1316 # 0.4.0
-ATC_ROUTER=1eeb0509a90494dc8618c5cd034ca4be231bb344 # 1.6.1
+ATC_ROUTER=ffd11db657115769bf94f0c4f915f98300bc26b6 # 1.6.2
 
 KONG_MANAGER=nightly
 NGX_WASM_MODULE=3bd94e61c55415ccfb0f304fa51143a7d630d6ae

build/libexpat/repositories.bzl

@@ -14,7 +14,7 @@ def libexpat_repositories():
         http_archive,
         name = "libexpat",
         url = "https://github.com/libexpat/libexpat/releases/download/" + tag + "/expat-" + version + ".tar.gz",
-        sha256 = "6b902ab103843592be5e99504f846ec109c1abb692e85347587f237a4ffa1033",
+        sha256 = "d4cf38d26e21a56654ffe4acd9cd5481164619626802328506a2869afab29ab3",
         strip_prefix = "expat-" + version,
         build_file = "//build/libexpat:BUILD.libexpat.bazel",
     )

build/openresty/patches/lua-cjson-2.1.0.13_01-error-on-t_end.patch

@@ -0,0 +1,26 @@
+From e1fca089680e76896744ec2f25219dd705fe21da Mon Sep 17 00:00:00 2001
+From: Wangchong Zhou <wangchong@konghq.com>
+Date: Wed, 17 Apr 2024 18:00:10 +0800
+Subject: [PATCH 1/4] bugfix: throw error if T_END found in the middle of input
+
+---
+ lua_cjson.c    | 4 ++++
+ tests/test.lua | 5 +++++
+ 2 files changed, 9 insertions(+)
+
+diff --git a/bundle/lua-cjson-2.1.0.13/lua_cjson.c b/bundle/lua-cjson-2.1.0.13/lua_cjson.c
+index 363466c..7343f32 100644
+--- a/bundle/lua-cjson-2.1.0.13/lua_cjson.c
++++ b/bundle/lua-cjson-2.1.0.13/lua_cjson.c
+@@ -1437,6 +1437,10 @@ static int json_decode(lua_State *l)
+     if (token.type != T_END)
+         json_throw_parse_error(l, &json, "the end", &token);
+
++    /* Make sure T_END (\x00) doesn't occur at middle of input */
++    if (json.data + json_len > json.ptr)
++        json_throw_parse_error(l, &json, "EOF", &token);
++
+     strbuf_free(json.tmp);
+
+     return 1;
+

changelog/unreleased/kong/bump-atc-router.yml

@@ -1,3 +1,3 @@
-message: "Bumped atc-router from v1.6.0 to v1.6.1"
+message: "Bumped atc-router from v1.6.0 to v1.6.2"
 type: dependency
 scope: Core

changelog/unreleased/kong/bump-libexpat.yml

@@ -0,0 +1,3 @@
+message: "Bumped libexpat to 2.6.2"
+type: dependency
+scope: Core

changelog/unreleased/kong/bump-lua-kong-nginx-module.yml

@@ -1,3 +1,4 @@
-message: "Bumped lua-kong-nginx-module from 0.8.0 to 0.10.0"
+message: |
+  Bumped lua-kong-nginx-module from 0.8.0 to 0.11.0
 type: dependency
 scope: Core

changelog/unreleased/kong/bump-lua-resty-acme.yml

@@ -0,0 +1,3 @@
+message: "Bumped lua-resty-acme to 0.13.0"
+type: dependency
+scope: Core

changelog/unreleased/kong/bump-lua-resty-http-0.17.2.yml

@@ -0,0 +1,2 @@
+message: Bump lua-resty-http to 0.17.2.
+type: dependency

changelog/unreleased/kong/bump-lua-resty-openssl.yml

@@ -1,3 +1,3 @@
-message: "Bumped lua-resty-openssl to 1.2.1"
+message: Bumped lua-resty-openssl from 1.2.0 to 1.3.1
 type: dependency
 scope: Core

changelog/unreleased/kong/fix-cjson-t-end.yml

@@ -0,0 +1,3 @@
+message: |
+  Improve the robustness of lua-cjson when handling unexpected input. 
+type: dependency

changelog/unreleased/kong/fix-dbless-duplicate-target-error.yml

@@ -0,0 +1,3 @@
+message: "Fixed an issue wherein `POST /config?flatten_errors=1` could not return a proper response if the input included duplicate upstream targets"
+type: bugfix
+scope: Core

changelog/unreleased/kong/fix-vault-secret-update-without-ttl.yml

@@ -0,0 +1,3 @@
+message: Fixed a bug that allowed vault secrets to refresh even when they had no TTL set.
+type: bugfix
+scope: Core

changelog/unreleased/kong/fix-wasm-disable-pwm-lua-resolver.yml

@@ -0,0 +1,4 @@
+message: |
+   Disable usage of the Lua DNS resolver from proxy-wasm by default.
+type: bugfix
+scope: Configuration

changelog/unreleased/kong/update-ai-proxy-telemetry.yml

@@ -1,3 +1,3 @@
-message: Update telemetry collection for AI Plugins to allow multiple instances data to be set for the same request.
+message: Update telemetry collection for AI Plugins to allow multiple plugins data to be set for the same request.
 type: bugfix
 scope: Core

kong-3.7.0-0.rockspec

@@ -16,7 +16,7 @@ dependencies = {
   "luasec == 1.3.2",
   "luasocket == 3.0-rc1",
   "penlight == 1.14.0",
-  "lua-resty-http == 0.17.1",
+  "lua-resty-http == 0.17.2",
   "lua-resty-jit-uuid == 0.0.7",
   "lua-ffi-zlib == 0.6",
   "multipart == 0.5.9",
@@ -34,10 +34,10 @@ dependencies = {
   "lua-resty-healthcheck == 3.0.1",
   "lua-messagepack == 0.5.4",
   "lua-resty-aws == 1.4.1",
-  "lua-resty-openssl == 1.2.1",
+  "lua-resty-openssl == 1.3.1",
   "lua-resty-counter == 0.2.1",
   "lua-resty-ipmatcher == 0.6.1",
-  "lua-resty-acme == 0.12.0",
+  "lua-resty-acme == 0.13.0",
   "lua-resty-session == 4.0.5",
   "lua-resty-timer-ng == 0.2.7",
   "lpeg == 1.1.0",

kong/clustering/compat/checkers.lua

@@ -47,6 +47,7 @@ local compatible_checkers = {
       return has_update
     end,
   },
+
   { 3005000000, --[[ 3.5.0.0 ]]
     function(config_table, dp_version, log_suffix)
       local has_update

kong/clustering/compat/init.lua

@@ -402,4 +402,42 @@ function _M.update_compatible_payload(payload, dp_version, log_suffix)
 end
 
 
+-- If mixed config is detected and a 3.6 or lower DP is attached to the CP,
+-- no config will be sent at all
+function _M.check_mixed_route_entities(payload, dp_version, flavor)
+  if flavor ~= "expressions" then
+    return true
+  end
+
+  -- CP runs with 'expressions' flavor
+
+  local dp_version_num = version_num(dp_version)
+
+  if dp_version_num >= 3007000000 then -- [[ 3.7.0.0 ]]
+    return true
+  end
+
+  local routes = payload["config_table"].routes or {}
+  local routes_n = #routes
+  local count = 0   -- expression route count
+
+  for i = 1, routes_n do
+    local r = routes[i]
+
+    -- expression should be a string
+    if r.expression and r.expression ~= ngx.null then
+      count = count + 1
+    end
+  end
+
+  if count == routes_n or   -- all are expression only routes
+     count == 0             -- all are traditional routes
+  then
+    return true
+  end
+
+  return false, dp_version .. " does not support mixed mode route"
+end
+
+
 return _M

kong/clustering/control_plane.lua

@@ -39,6 +39,7 @@ local sleep = ngx.sleep
 
 local plugins_list_to_map = compat.plugins_list_to_map
 local update_compatible_payload = compat.update_compatible_payload
+local check_mixed_route_entities = compat.check_mixed_route_entities
 local deflate_gzip = require("kong.tools.gzip").deflate_gzip
 local yield = require("kong.tools.yield").yield
 local connect_dp = clustering_utils.connect_dp
@@ -432,6 +433,15 @@ function _M:handle_cp_websocket(cert)
         goto continue
       end
 
+      ok, err = check_mixed_route_entities(self.reconfigure_payload, dp_version,
+                                           kong and kong.configuration and
+                                           kong.configuration.router_flavor)
+      if not ok then
+        ngx_log(ngx_WARN, _log_prefix, "unable to send updated configuration to data plane: ", err, log_suffix)
+
+        goto continue
+      end
+
       local _, deflated_payload, err = update_compatible_payload(self.reconfigure_payload, dp_version, log_suffix)
 
       if not deflated_payload then -- no modification or err, use the cached payload

kong/conf_loader/init.lua

@@ -563,7 +563,8 @@ local function load(path, custom_conf, opts)
     -- set it as such in kong_defaults, because it can only be used if wasm is
     -- _also_ enabled. We inject it here if the user has not opted to set it
     -- themselves.
-    add_wasm_directive("nginx_http_proxy_wasm_lua_resolver", "on")
+    -- TODO: as a temporary compatibility fix, we are forcing it to 'off'.
+    add_wasm_directive("nginx_http_proxy_wasm_lua_resolver", "off")
 
     -- wasm vm properties are inherited from previously set directives
     if conf.lua_ssl_trusted_certificate and #conf.lua_ssl_trusted_certificate >= 1 then

kong/db/errors.lua

@@ -795,12 +795,19 @@ do
   ---@param err_t       table
   ---@param flattened   table
   local function add_entity_errors(entity_type, entity, err_t, flattened)
-    if type(err_t) ~= "table" or nkeys(err_t) == 0 then
+    local err_type = type(err_t)
+
+    -- promote error strings to `@entity` type errors
+    if err_type == "string" then
+      err_t = { ["@entity"] = err_t }
+
+    elseif err_type ~= "table" or nkeys(err_t) == 0 then
       return
+    end
 
     -- this *should* be unreachable, but it's relatively cheap to guard against
     -- compared to everything else we're doing in this code path
-    elseif type(entity) ~= "table" then
+    if type(entity) ~= "table" then
       log(WARN, "could not parse ", entity_type, " errors for non-table ",
                 "input: '", tostring(entity), "'")
       return
@@ -1033,13 +1040,7 @@ do
       for i, err_t_i in drain(section_errors) do
         local entity = entities[i]
 
-
-        -- promote error strings to `@entity` type errors
-        if type(err_t_i) == "string" then
-          err_t_i = { ["@entity"] = err_t_i }
-        end
-
-        if type(entity) == "table" and type(err_t_i) == "table" then
+        if type(entity) == "table" then
           add_entity_errors(entity_type, entity, err_t_i, flattened)
 
         else

kong/db/schema/others/declarative_config.lua

@@ -335,7 +335,7 @@ local function uniqueness_error_msg(entity, key, value)
 end
 
 
-local function populate_references(input, known_entities, by_id, by_key, expected, errs, parent_entity)
+local function populate_references(input, known_entities, by_id, by_key, expected, errs, parent_entity, parent_idx)
   for _, entity in ipairs(known_entities) do
     yield(true)
 
@@ -363,7 +363,7 @@ local function populate_references(input, known_entities, by_id, by_key, expecte
     for i, item in ipairs(input[entity]) do
       yield(true)
 
-      populate_references(item, known_entities, by_id, by_key, expected, errs, entity)
+      populate_references(item, known_entities, by_id, by_key, expected, errs, entity, i)
 
       local item_id = DeclarativeConfig.pk_string(entity_schema, item)
       local key = use_key and item[endpoint_key]
@@ -381,8 +381,23 @@ local function populate_references(input, known_entities, by_id, by_key, expecte
       if item_id then
         by_id[entity] = by_id[entity] or {}
         if (not failed) and by_id[entity][item_id] then
-          errs[entity] = errs[entity] or {}
-          errs[entity][i] = uniqueness_error_msg(entity, "primary key", item_id)
+          local err_t
+
+          if parent_entity and parent_idx then
+            errs[parent_entity]                     = errs[parent_entity] or {}
+            errs[parent_entity][parent_idx]         = errs[parent_entity][parent_idx] or {}
+            errs[parent_entity][parent_idx][entity] = errs[parent_entity][parent_idx][entity] or {}
+
+            -- e.g. errs["upstreams"][5]["targets"]
+            err_t = errs[parent_entity][parent_idx][entity]
+
+          else
+            errs[entity] = errs[entity] or {}
+            err_t = errs[entity]
+          end
+
+          err_t[i] = uniqueness_error_msg(entity, "primary key", item_id)
+
         else
           by_id[entity][item_id] = item
           table.insert(by_id[entity], item_id)