From dc77461c0c55fe36042b4e30ead620d7bb7ca53f Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Mon, 4 Nov 2024 15:52:07 +0800 Subject: [PATCH 1/7] fix(sync/lmdb): construct right pk string for entity with multiple primary keys For the entity with multiple primary keys, like consumer_group_consumers, the pk_string() function construct incorrect pk string like `table: 0x028a49eda0:table: 0x0289c257b8`. Because it does not extract the internal id value. Here, we fix it with original lmdb logic get_cache_key_value(). Note this method directly gets id field name. --- kong/db/schema/others/declarative_config.lua | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index 42b165f7e7ff..666b340bb898 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -60,9 +60,18 @@ do end CACHED_OUT:clear() + + -- The logic comes from get_cache_key_value(), which uses `id` directly to + -- extract foreign key. for i = 1, count do local k = primary_key[i] - insert(CACHED_OUT, tostring(object[k])) + local v = object[k] + + if type(v) == "table" and schema.fields[k].type == "foreign" then + v = v.id + end + + insert(CACHED_OUT, tostring(v or "")) end return concat(CACHED_OUT, ":") From 4656916c61df01f0922da58ec6f3ebc00baebd73 Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 5 Nov 2024 13:36:46 +0800 Subject: [PATCH 2/7] fix(sync): don't use an upvalue request_aware_table to construct pk_string (#10623) * fix(sync): don't use an upvalue request_aware_table to construct pk_string * fix: use local upvalue {} --- kong/db/schema/others/declarative_config.lua | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index 666b340bb898..33ac6a184fd7 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -41,9 +41,11 @@ local foreign_children = {} do local tb_nkeys = require("table.nkeys") - local request_aware_table = require("kong.tools.request_aware_table") + local tb_clear = require("table.clear") - local CACHED_OUT + -- We couldn't use "kong.tools.request_aware_table" as an upvalue table here + -- because its :clear() couldn't actually clear its contents in debug mode. + local CACHED_OUT = {} -- Generate a stable and unique string key from primary key defined inside -- schema, supports both non-composite and composite primary keys @@ -55,11 +57,7 @@ do return tostring(object[primary_key[1]]) end - if not CACHED_OUT then - CACHED_OUT = request_aware_table.new() - end - - CACHED_OUT:clear() + tb_clear(CACHED_OUT) -- The logic comes from get_cache_key_value(), which uses `id` directly to -- extract foreign key. From 03f2a1c4c26b3fd1912b3d2127e76ce79a8fad43 Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 5 Nov 2024 15:44:20 +0800 Subject: [PATCH 3/7] fix(sync): use local table to contain pk strings in pk_string() --- kong/db/schema/others/declarative_config.lua | 8 ++------ kong/tools/request_aware_table.lua | 4 ++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index 33ac6a184fd7..dc24264fffa2 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -41,11 +41,6 @@ local foreign_children = {} do local tb_nkeys = require("table.nkeys") - local tb_clear = require("table.clear") - - -- We couldn't use "kong.tools.request_aware_table" as an upvalue table here - -- because its :clear() couldn't actually clear its contents in debug mode. - local CACHED_OUT = {} -- Generate a stable and unique string key from primary key defined inside -- schema, supports both non-composite and composite primary keys @@ -57,10 +52,11 @@ do return tostring(object[primary_key[1]]) end - tb_clear(CACHED_OUT) + local CACHED_OUT = {} -- The logic comes from get_cache_key_value(), which uses `id` directly to -- extract foreign key. + -- TODO: extract primary key recursively using pk_string(), KAG-5750 for i = 1, count do local k = primary_key[i] local v = object[k] diff --git a/kong/tools/request_aware_table.lua b/kong/tools/request_aware_table.lua index beaa08f4b6e9..7e90419614f9 100644 --- a/kong/tools/request_aware_table.lua +++ b/kong/tools/request_aware_table.lua @@ -1,5 +1,9 @@ --- NOTE: tool is designed to assist with **detecting** request contamination -- issues on CI, during test runs. It does not offer security safeguards. +-- +-- In debug mode, +-- 1. `:clear` could not clear elements inserted by `table.insert()` +-- 2. `table.concat()` couldn't work for elements assigned using `indexing` local table_new = require("table.new") local table_clear = require("table.clear") From 5b1a72029b5365de05ebde01cce4bac32649e91c Mon Sep 17 00:00:00 2001 From: Xiaochen Wang Date: Tue, 5 Nov 2024 16:24:28 +0800 Subject: [PATCH 4/7] fix comment: add TODO tag --- kong/tools/request_aware_table.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kong/tools/request_aware_table.lua b/kong/tools/request_aware_table.lua index 7e90419614f9..3a1cc8894373 100644 --- a/kong/tools/request_aware_table.lua +++ b/kong/tools/request_aware_table.lua @@ -1,7 +1,7 @@ --- NOTE: tool is designed to assist with **detecting** request contamination -- issues on CI, during test runs. It does not offer security safeguards. -- --- In debug mode, +-- TODO: need to resolve the following issues in debug mode, -- 1. `:clear` could not clear elements inserted by `table.insert()` -- 2. `table.concat()` couldn't work for elements assigned using `indexing` From e25a18882b73d2ada71de55b06a357ea90a94ba6 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 5 Nov 2024 17:11:06 +0800 Subject: [PATCH 5/7] use tablepool --- kong/db/schema/others/declarative_config.lua | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index dc24264fffa2..74e9294b396a 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -41,6 +41,7 @@ local foreign_children = {} do local tb_nkeys = require("table.nkeys") + local tb_pool = require("tablepool") -- Generate a stable and unique string key from primary key defined inside -- schema, supports both non-composite and composite primary keys @@ -52,7 +53,8 @@ do return tostring(object[primary_key[1]]) end - local CACHED_OUT = {} + -- get a table for reuse purpose + local CACHED_OUT = tb_pool.fetch("dc_cached_pk", 2, 0) -- The logic comes from get_cache_key_value(), which uses `id` directly to -- extract foreign key. @@ -68,7 +70,12 @@ do insert(CACHED_OUT, tostring(v or "")) end - return concat(CACHED_OUT, ":") + local str = concat(CACHED_OUT, ":") + + -- releae table for next usage + tb_pool.release("dc_cached_pk", CACHED_OUT) + + return str end end From 54f3ba9d3516859afb87a458b7a248e3d13653a0 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 5 Nov 2024 17:13:38 +0800 Subject: [PATCH 6/7] typo fix --- kong/db/schema/others/declarative_config.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index 74e9294b396a..f244cad3a39a 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -72,7 +72,7 @@ do local str = concat(CACHED_OUT, ":") - -- releae table for next usage + -- release table for next usage tb_pool.release("dc_cached_pk", CACHED_OUT) return str From e1251b369c8b814a8d88b19b504d337df8f75695 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Tue, 5 Nov 2024 17:19:59 +0800 Subject: [PATCH 7/7] string.buffer --- kong/db/schema/others/declarative_config.lua | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/kong/db/schema/others/declarative_config.lua b/kong/db/schema/others/declarative_config.lua index f244cad3a39a..7cae8ffe3c64 100644 --- a/kong/db/schema/others/declarative_config.lua +++ b/kong/db/schema/others/declarative_config.lua @@ -41,7 +41,7 @@ local foreign_children = {} do local tb_nkeys = require("table.nkeys") - local tb_pool = require("tablepool") + local buffer = require("string.buffer") -- Generate a stable and unique string key from primary key defined inside -- schema, supports both non-composite and composite primary keys @@ -53,8 +53,7 @@ do return tostring(object[primary_key[1]]) end - -- get a table for reuse purpose - local CACHED_OUT = tb_pool.fetch("dc_cached_pk", 2, 0) + local buf = buffer.new() -- The logic comes from get_cache_key_value(), which uses `id` directly to -- extract foreign key. @@ -67,15 +66,14 @@ do v = v.id end - insert(CACHED_OUT, tostring(v or "")) - end - - local str = concat(CACHED_OUT, ":") + buf:put(tostring(v or "")) - -- release table for next usage - tb_pool.release("dc_cached_pk", CACHED_OUT) + if i < count then + buf:put(":") + end + end - return str + return buf:get() end end