[cherry release/3.4.x] cherry-picking important fixes to 34x

changelog/unreleased/kong/11464.yml

@@ -0,0 +1,7 @@
+message: Fix an issue that the TTL of the key-auth plugin didnt work in DB-less and Hybrid mode.
+type: bugfix
+scope: Core
+prs:
+  - 11464
+jiras:
+  - "FTI-4512"

changelog/unreleased/kong/11566.yml

@@ -0,0 +1,7 @@
+message: "use deep copies of Route, Service, and Consumer objects when log serializing"
+type: bugfix
+scope: PDK
+prs:
+  - 11566
+jiras:
+  - "FTI-5357"

changelog/unreleased/kong/11613.yml

@@ -0,0 +1,4 @@
+message: "Bumped lua-resty-aws from 1.3.2 to 1.3.5"
+type: dependency
+prs:
+  - 11613

changelog/unreleased/kong/acl_cache_warmup.yml

@@ -0,0 +1,5 @@
+message: Fix cache warmup mechanism not working in `acls` plugin groups config entity scenario.
+type: bugfix
+scope: Core
+prs:
+  - 11414

changelog/unreleased/kong/rate-limiting-fix-redis-sync-rate.yml

@@ -0,0 +1,3 @@
+message: "**Rate Limiting**: fix to provide better accuracy in counters when sync_rate is used with the redis policy."
+type: bugfix
+scope: Plugin

changelog/unreleased/kong/response_status_code.yml

@@ -0,0 +1,3 @@
+message: Fix an issue that response status code is not real upstream status when using kong.response function.
+type: bugfix
+scope: Core

changelog/unreleased/kong/rl-shared-sync-timer.yml

@@ -0,0 +1,3 @@
+message: "**Rate Limiting**: fix an issuer where all counters are synced to the same DB at the same rate."
+type: bugfix
+scope: Plugin

kong-3.4.3-0.rockspec

@@ -33,7 +33,7 @@ dependencies = {
   "lua-protobuf == 0.5.0",
   "lua-resty-healthcheck == 1.6.3",
   "lua-messagepack == 0.5.2",
-  "lua-resty-aws == 1.3.2",
+  "lua-resty-aws == 1.3.5",
   "lua-resty-openssl == 0.8.23",
   "lua-resty-counter == 0.2.1",
   "lua-resty-ipmatcher == 0.6.1",

kong/cache/warmup.lua

@@ -1,6 +1,10 @@
 local utils = require "kong.tools.utils"
 local constants = require "kong.constants"
 local buffer = require "string.buffer"
+local acl_groups
+if utils.load_module_if_exists("kong.plugins.acl.groups") then
+  acl_groups = require "kong.plugins.acl.groups"
+end
 
 
 local cache_warmup = {}
@@ -136,6 +140,14 @@ function cache_warmup.single_dao(dao)
     if not ok then
       return nil, err
     end
+
+    if entity_name == "acls" and acl_groups ~= nil then
+      log(NOTICE, "warmup acl groups cache for consumer id: ", entity.consumer.id , "...")
+      local _, err = acl_groups.warmup_groups_cache(entity.consumer.id)
+      if err then
+        log(NOTICE, "warmup acl groups cache for consumer id: ", entity.consumer.id , " err: ", err)
+      end
+    end
   end
 
   if entity_name == "services" and host_count > 0 then

kong/db/dao/init.lua

@@ -283,6 +283,12 @@ local function validate_options_value(self, options)
     end
   end
 
+  if options.export ~= nil then
+    if type(options.export) ~= "boolean" then
+      errors.export = "must be a boolean"
+    end
+  end
+
   if next(errors) then
     return nil, errors
   end
@@ -1103,6 +1109,21 @@ function DAO:each(size, options)
 end
 
 
+function DAO:each_for_export(size, options)
+  if self.strategy.schema.ttl then
+    if not options then
+      options = get_pagination_options(self, options)
+    else
+      options = utils.cycle_aware_deep_copy(options, true)
+    end
+
+    options.export = true
+  end
+
+  return self:each(size, options)
+end
+
+
 function DAO:insert(entity, options)
   validate_entity_type(entity)
 

kong/db/declarative/export.lua

@@ -142,7 +142,7 @@ local function export_from_db_impl(emitter, skip_ws, skip_disabled_entities, exp
     if db[name].pagination then
       page_size = db[name].pagination.max_page_size
     end
-    for row, err in db[name]:each(page_size, GLOBAL_QUERY_OPTS) do
+    for row, err in db[name]:each_for_export(page_size, GLOBAL_QUERY_OPTS) do
       if not row then
         end_transaction(db)
         kong.log.err(err)

kong/db/schema/others/declarative_config.lua

@@ -830,6 +830,10 @@ local function flatten(self, input)
         end
       end
 
+      if schema.ttl and entry.ttl and entry.ttl ~= null then
+        flat_entry.ttl = entry.ttl
+      end
+
       entities[entity][id] = flat_entry
     end
   end

kong/db/strategies/off/init.lua

@@ -54,6 +54,19 @@ local function ws(schema, options)
 end
 
 
+local function process_ttl_field(entity)
+  if entity and entity.ttl and entity.ttl ~= null then
+    local ttl_value = entity.ttl - ngx.time()
+    if ttl_value > 0 then
+      entity.ttl = ttl_value
+    else
+      entity = nil  -- do not return the expired entity
+    end
+  end
+  return entity
+end
+
+
 -- Returns a dict of entity_ids tagged according to the given criteria.
 -- Currently only the following kinds of keys are supported:
 -- * A key like `services|<ws_id>|@list` will only return service keys
@@ -157,6 +170,7 @@ local function page_for_key(self, key, size, offset, options)
   yield()
 
   local ret = {}
+  local ret_idx = 1
   local schema = self.schema
   local schema_name = schema.name
 
@@ -194,7 +208,14 @@ local function page_for_key(self, key, size, offset, options)
       return nil, "stale data detected while paginating"
     end
 
-    ret[i - offset + 1] = schema:process_auto_fields(item, "select", true, PROCESS_AUTO_FIELDS_OPTS)
+    if schema.ttl then
+      item = process_ttl_field(item)
+    end
+
+    if item then
+      ret[ret_idx] = schema:process_auto_fields(item, "select", true, PROCESS_AUTO_FIELDS_OPTS)
+      ret_idx = ret_idx + 1
+    end
   end
 
   if offset then
@@ -211,6 +232,13 @@ local function select_by_key(schema, key)
     return nil, err
   end
 
+  if schema.ttl then
+    entity = process_ttl_field(entity)
+    if not entity then
+      return nil
+    end
+  end
+
   entity = schema:process_auto_fields(entity, "select", true, PROCESS_AUTO_FIELDS_OPTS)
 
   return entity

kong/db/strategies/postgres/connector.lua

@@ -567,7 +567,7 @@ function _mt:query(sql, operation)
       -- we cannot cleanup the connection
       ngx.log(ngx.ERR, "failed to disconnect: ", err)
     end
-    self.store_connection(nil, operation)
+    self:store_connection(nil, operation)
 
   elseif is_new_conn then
     local keepalive_timeout = self:get_keepalive_timeout(operation)

kong/db/strategies/postgres/init.lua

@@ -481,6 +481,10 @@ local function page(self, size, token, foreign_key, foreign_entity_name, options
     statement_name = "page" .. suffix
   end
 
+  if options and options.export then
+    statement_name = statement_name .. "_for_export"
+  end
+
   if token then
     local token_decoded = decode_base64(token)
     if not token_decoded then
@@ -1022,6 +1026,7 @@ function _M.new(connector, schema, errors)
     ws_id_select_where = "(" .. ws_id_escaped .. " = $0)"
   end
 
+  local select_for_export_expressions
   local ttl_select_where
   if has_ttl then
     fields_hash.ttl = { timestamp = true }
@@ -1030,6 +1035,13 @@ function _M.new(connector, schema, errors)
     insert(insert_expressions, "$" .. #insert_names)
     insert(insert_columns, ttl_escaped)
 
+    select_for_export_expressions = concat {
+      select_expressions, ",",
+      "FLOOR(EXTRACT(EPOCH FROM (",
+        ttl_escaped, " AT TIME ZONE 'UTC'",
+      "))) AS ", ttl_escaped
+    }
+
     select_expressions = concat {
       select_expressions, ",",
       "FLOOR(EXTRACT(EPOCH FROM (",
@@ -1078,6 +1090,7 @@ function _M.new(connector, schema, errors)
   self.statements["truncate_global"] = self.statements["truncate"]
 
   local add_statement
+  local add_statement_for_export
   do
     local function add(name, opts, add_ws)
       local orig_argn = opts.argn
@@ -1106,6 +1119,14 @@ function _M.new(connector, schema, errors)
       add(name .. "_global", opts, false)
       add(name, opts, true)
     end
+
+    add_statement_for_export = function(name, opts)
+      add_statement(name, opts)
+      if has_ttl then
+        opts.code[2] = select_for_export_expressions
+        add_statement(name .. "_for_export", opts)
+      end
+    end
   end
 
   add_statement("insert", {
@@ -1181,7 +1202,7 @@ function _M.new(connector, schema, errors)
     }
   })
 
-  add_statement("page_first", {
+  add_statement_for_export("page_first", {
     operation = "read",
     argn = { LIMIT },
     argv = single_args,
@@ -1196,7 +1217,7 @@ function _M.new(connector, schema, errors)
     }
   })
 
-  add_statement("page_next", {
+  add_statement_for_export("page_next", {
     operation = "read",
     argn = page_next_names,
     argv = page_next_args,
@@ -1246,7 +1267,7 @@ function _M.new(connector, schema, errors)
 
       local statement_name = "page_for_" .. foreign_entity_name
 
-      add_statement(statement_name .. "_first", {
+      add_statement_for_export(statement_name .. "_first", {
         operation = "read",
         argn = argn_first,
         argv = argv_first,
@@ -1262,7 +1283,7 @@ function _M.new(connector, schema, errors)
         }
       })
 
-      add_statement(statement_name .. "_next", {
+      add_statement_for_export(statement_name .. "_next", {
         operation = "read",
         argn = argn_next,
         argv = argv_next,
@@ -1297,7 +1318,7 @@ function _M.new(connector, schema, errors)
 
     for cond, op in pairs({["_and"] = "@>", ["_or"] = "&&"}) do
 
-      add_statement("page_by_tags" .. cond .. "_first", {
+      add_statement_for_export("page_by_tags" .. cond .. "_first", {
         operation = "read",
         argn = argn_first,
         argv = {},
@@ -1313,7 +1334,7 @@ function _M.new(connector, schema, errors)
         },
       })
 
-      add_statement("page_by_tags" .. cond .. "_next", {
+      add_statement_for_export("page_by_tags" .. cond .. "_next", {
         operation = "read",
         argn = argn_next,
         argv = {},

kong/init.lua

@@ -1455,7 +1455,7 @@ do
     local res = ngx.location.capture("/kong_buffered_http", options)
     if res.truncated and options.method ~= ngx.HTTP_HEAD then
       ctx.KONG_PHASE = PHASES.error
-      ngx.status = 502
+      ngx.status = res.status or 502
 
       if has_timing then
         req_dyn_hook_run_hooks(ctx, "timing", "after:response")

kong/pdk/log.lua

@@ -16,7 +16,7 @@ local inspect = require "inspect"
 local ngx_ssl = require "ngx.ssl"
 local phase_checker = require "kong.pdk.private.phases"
 local utils = require "kong.tools.utils"
-
+local cycle_aware_deep_copy = utils.cycle_aware_deep_copy
 
 local sub = string.sub
 local type = type
@@ -810,7 +810,7 @@ do
         end
       end
 
-      -- The value of upstream_status is a string, and status codes may be 
+      -- The value of upstream_status is a string, and status codes may be
       -- seperated by comma or grouped by colon, according to
       -- the nginx doc: http://nginx.org/en/docs/http/ngx_http_upstream_module.html#upstream_status
       local upstream_status = var.upstream_status or ""
@@ -841,9 +841,9 @@ do
         },
         tries = (ctx.balancer_data or {}).tries,
         authenticated_entity = build_authenticated_entity(ctx),
-        route = ctx.route,
-        service = ctx.service,
-        consumer = ctx.authenticated_consumer,
+        route = cycle_aware_deep_copy(ctx.route),
+        service = cycle_aware_deep_copy(ctx.service),
+        consumer = cycle_aware_deep_copy(ctx.authenticated_consumer),
         client_ip = var.remote_addr,
         started_at = okong.request.get_start_time(),
       }
@@ -882,9 +882,9 @@ do
         },
         tries = (ctx.balancer_data or {}).tries,
         authenticated_entity = build_authenticated_entity(ctx),
-        route = ctx.route,
-        service = ctx.service,
-        consumer = ctx.authenticated_consumer,
+        route = cycle_aware_deep_copy(ctx.route),
+        service = cycle_aware_deep_copy(ctx.service),
+        consumer = cycle_aware_deep_copy(ctx.authenticated_consumer),
         client_ip = var.remote_addr,
         started_at = okong.request.get_start_time(),
       }

kong/plugins/acl/groups.lua

@@ -196,11 +196,22 @@ local function group_in_groups(groups_to_check, groups)
   end
 end
 
+local function warmup_groups_cache(consumer_id)
+  local cache_key = kong.db.acls:cache_key(consumer_id)
+  local _, err = kong.cache:get(cache_key, nil,
+                                         load_groups_into_memory,
+                                         { id = consumer_id })
+  if err then
+    return nil, err
+  end
+end
+
 
 return {
   get_current_consumer_id = get_current_consumer_id,
   get_consumer_groups = get_consumer_groups,
   get_authenticated_groups = get_authenticated_groups,
   consumer_in_groups = consumer_in_groups,
   group_in_groups = group_in_groups,
+  warmup_groups_cache = warmup_groups_cache,
 }