diff --git a/build/dockerfiles/entrypoint.sh b/build/dockerfiles/entrypoint.sh index 4aa71dedaa1d..11a2e530ecea 100755 --- a/build/dockerfiles/entrypoint.sh +++ b/build/dockerfiles/entrypoint.sh @@ -46,11 +46,12 @@ if [[ "$1" == "kong" ]]; then # remove all dangling sockets in $PREFIX dir before starting Kong LOGGED_SOCKET_WARNING=0 - for localfile in "$PREFIX"/*; do + socket_path=$PREFIX/sockets + for localfile in "$socket_path"/*; do if [ -S "$localfile" ]; then if (( LOGGED_SOCKET_WARNING == 0 )); then printf >&2 'WARN: found dangling unix sockets in the prefix directory ' - printf >&2 '(%q) ' "$PREFIX" + printf >&2 '(%q) ' "$socket_path" printf >&2 'while preparing to start Kong. This may be a sign that Kong ' printf >&2 'was previously shut down uncleanly or is in an unknown state ' printf >&2 'and could require further investigation.\n' diff --git a/changelog/unreleased/kong/move-sockets-to-subdir.yml b/changelog/unreleased/kong/move-sockets-to-subdir.yml new file mode 100644 index 000000000000..37fdd5d10a85 --- /dev/null +++ b/changelog/unreleased/kong/move-sockets-to-subdir.yml @@ -0,0 +1,3 @@ +message: Moved internal Unix sockets to a subdirectory (`sockets`) of the Kong prefix. +type: bugfix +scope: Core diff --git a/kong/clustering/utils.lua b/kong/clustering/utils.lua index 1f3bdc2db621..8598ff3e51ce 100644 --- a/kong/clustering/utils.lua +++ b/kong/clustering/utils.lua @@ -25,8 +25,8 @@ local _log_prefix = "[clustering] " local KONG_VERSION = kong.version -local prefix = kong.configuration.prefix or require("pl.path").abspath(ngx.config.prefix()) -local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock", prefix) +local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock", + kong.configuration.socket_path) local _M = {} diff --git a/kong/cmd/start.lua b/kong/cmd/start.lua index 75c0c7b0af8a..6bc2dd97b416 100644 --- a/kong/cmd/start.lua +++ b/kong/cmd/start.lua @@ -13,11 +13,11 @@ local function is_socket(path) return lfs.attributes(path, "mode") == "socket" end -local function cleanup_dangling_unix_sockets(prefix) +local function cleanup_dangling_unix_sockets(socket_path) local found = {} - for child in lfs.dir(prefix) do - local path = prefix .. "/" .. child + for child in lfs.dir(socket_path) do + local path = socket_path .. "/" .. child if is_socket(path) then table.insert(found, path) end @@ -31,7 +31,7 @@ local function cleanup_dangling_unix_sockets(prefix) "preparing to start Kong. This may be a sign that Kong was " .. "previously shut down uncleanly or is in an unknown state and " .. "could require further investigation.", - prefix) + socket_path) log.warn("Attempting to remove dangling sockets before starting Kong...") @@ -59,7 +59,7 @@ local function execute(args) assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, nil, args.nginx_conf_flags)) - cleanup_dangling_unix_sockets(conf.prefix) + cleanup_dangling_unix_sockets(conf.socket_path) _G.kong = kong_global.new() kong_global.init_pdk(_G.kong, conf) diff --git a/kong/cmd/utils/prefix_handler.lua b/kong/cmd/utils/prefix_handler.lua index b1e6557f4ac5..74268b139bb3 100644 --- a/kong/cmd/utils/prefix_handler.lua +++ b/kong/cmd/utils/prefix_handler.lua @@ -481,6 +481,13 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ return nil, kong_config.prefix .. " is not a directory" end + if not exists(kong_config.socket_path) then + local ok, err = makepath(kong_config.socket_path) + if not ok then + return nil, err + end + end + -- create directories in prefix for _, dir in ipairs {"logs", "pids"} do local ok, err = makepath(join(kong_config.prefix, dir)) diff --git a/kong/conf_loader/init.lua b/kong/conf_loader/init.lua index 48a893e2ed79..ac11ecb27b88 100644 --- a/kong/conf_loader/init.lua +++ b/kong/conf_loader/init.lua @@ -17,6 +17,7 @@ local pl_path = require "pl.path" local tablex = require "pl.tablex" local log = require "kong.cmd.utils.log" local env = require "kong.cmd.utils.env" +local constants = require "kong.constants" local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy @@ -482,6 +483,10 @@ local function load(path, custom_conf, opts) -- load absolute paths conf.prefix = abspath(conf.prefix) + -- The socket path is where we store listening unix sockets for IPC and private APIs. + -- It is derived from the prefix and is NOT intended to be user-configurable + conf.socket_path = pl_path.join(conf.prefix, constants.SOCKET_DIRECTORY) + if conf.lua_ssl_trusted_certificate and #conf.lua_ssl_trusted_certificate > 0 then diff --git a/kong/constants.lua b/kong/constants.lua index 63df1f2f5a47..33b8dcaa0aa5 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -280,6 +280,8 @@ local constants = { service = "upstream", } }, + + SOCKET_DIRECTORY = "sockets", } for _, v in ipairs(constants.CLUSTERING_SYNC_STATUS) do diff --git a/kong/global.lua b/kong/global.lua index b7a1bbc04ee0..f40c0b2c5892 100644 --- a/kong/global.lua +++ b/kong/global.lua @@ -168,28 +168,20 @@ function _GLOBAL.init_pdk(self, kong_config) end -function _GLOBAL.init_worker_events() +function _GLOBAL.init_worker_events(kong_config) -- Note: worker_events will not work correctly if required at the top of the file. -- It must be required right here, inside the init function local worker_events local opts - local configuration = kong.configuration - - -- `kong.configuration.prefix` is already normalized to an absolute path, - -- but `ngx.config.prefix()` is not - local prefix = configuration and - configuration.prefix or - require("pl.path").abspath(ngx.config.prefix()) - + local socket_path = kong_config.socket_path local sock = ngx.config.subsystem == "stream" and "stream_worker_events.sock" or "worker_events.sock" - local listening = "unix:" .. prefix .. "/" .. sock + local listening = "unix:" .. socket_path .. "/" .. sock - local max_payload_len = configuration and - configuration.worker_events_max_payload + local max_payload_len = kong_config.worker_events_max_payload if max_payload_len and max_payload_len > 65535 then -- default is 64KB ngx.log(ngx.WARN, @@ -203,9 +195,9 @@ function _GLOBAL.init_worker_events() listening = listening, -- unix socket for broker listening max_queue_len = 1024 * 50, -- max queue len for events buffering max_payload_len = max_payload_len, -- max payload size in bytes - enable_privileged_agent = configuration and configuration.dedicated_config_processing - and configuration.role == "data_plane" - or false + enable_privileged_agent = kong_config.dedicated_config_processing + and kong_config.role == "data_plane" + or false, } worker_events = require "resty.events.compat" diff --git a/kong/init.lua b/kong/init.lua index 343a68174e99..2a68f2acadf4 100644 --- a/kong/init.lua +++ b/kong/init.lua @@ -837,7 +837,7 @@ function Kong.init_worker() schema_state = nil - local worker_events, err = kong_global.init_worker_events() + local worker_events, err = kong_global.init_worker_events(kong.configuration) if not worker_events then stash_init_worker_error("failed to instantiate 'kong.worker_events' " .. "module: " .. err) diff --git a/kong/runloop/events.lua b/kong/runloop/events.lua index b8faba64cbee..7c4a7aaecf78 100644 --- a/kong/runloop/events.lua +++ b/kong/runloop/events.lua @@ -507,12 +507,18 @@ local stream_reconfigure_listener do local buffer = require "string.buffer" - -- `kong.configuration.prefix` is already normalized to an absolute path, - -- but `ngx.config.prefix()` is not - local PREFIX = kong and kong.configuration and - kong.configuration.prefix or - require("pl.path").abspath(ngx.config.prefix()) - local STREAM_CONFIG_SOCK = "unix:" .. PREFIX .. "/stream_config.sock" + -- this module may be loaded before `kong.configuration` is initialized + local socket_path = kong and kong.configuration + and kong.configuration.socket_path + + if not socket_path then + -- `kong.configuration.socket_path` is already normalized to an absolute + -- path, but `ngx.config.prefix()` is not + socket_path = require("pl.path").abspath(ngx.config.prefix() .. "/" + .. constants.SOCKET_DIRECTORY) + end + + local STREAM_CONFIG_SOCK = "unix:" .. socket_path .. "/stream_config.sock" local IS_HTTP_SUBSYSTEM = ngx.config.subsystem == "http" local function broadcast_reconfigure_event(data) diff --git a/kong/runloop/handler.lua b/kong/runloop/handler.lua index e1169795dd58..cd05327021a0 100644 --- a/kong/runloop/handler.lua +++ b/kong/runloop/handler.lua @@ -896,11 +896,9 @@ return { init_worker = { before = function() - -- TODO: PR #9337 may affect the following line - local prefix = kong.configuration.prefix or ngx.config.prefix() - - STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", prefix) - STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", prefix) + local socket_path = kong.configuration.socket_path + STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", socket_path) + STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", socket_path) log_level.init_worker() diff --git a/kong/templates/nginx.lua b/kong/templates/nginx.lua index 108d268f56be..4833d9218c17 100644 --- a/kong/templates/nginx.lua +++ b/kong/templates/nginx.lua @@ -83,7 +83,7 @@ stream { > if cluster_ssl_tunnel then server { - listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock; + listen unix:${{SOCKET_PATH}}/cluster_proxy_ssl_terminator.sock; proxy_pass ${{cluster_ssl_tunnel}}; proxy_ssl on; diff --git a/kong/templates/nginx_kong.lua b/kong/templates/nginx_kong.lua index 5692d040b422..c024a723a688 100644 --- a/kong/templates/nginx_kong.lua +++ b/kong/templates/nginx_kong.lua @@ -592,7 +592,7 @@ server { server { charset UTF-8; server_name kong_worker_events; - listen unix:${{PREFIX}}/worker_events.sock; + listen unix:${{SOCKET_PATH}}/worker_events.sock; access_log off; location / { content_by_lua_block { diff --git a/kong/templates/nginx_kong_stream.lua b/kong/templates/nginx_kong_stream.lua index 68a165110a80..4ff956caaf8f 100644 --- a/kong/templates/nginx_kong_stream.lua +++ b/kong/templates/nginx_kong_stream.lua @@ -94,7 +94,7 @@ server { > end > if stream_proxy_ssl_enabled then - listen unix:${{PREFIX}}/stream_tls_terminate.sock ssl proxy_protocol; + listen unix:${{SOCKET_PATH}}/stream_tls_terminate.sock ssl proxy_protocol; > end access_log ${{PROXY_STREAM_ACCESS_LOG}}; @@ -175,7 +175,7 @@ server { } server { - listen unix:${{PREFIX}}/stream_tls_passthrough.sock proxy_protocol; + listen unix:${{SOCKET_PATH}}/stream_tls_passthrough.sock proxy_protocol; access_log ${{PROXY_STREAM_ACCESS_LOG}}; error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}}; @@ -205,7 +205,7 @@ server { > if database == "off" then server { - listen unix:${{PREFIX}}/stream_config.sock; + listen unix:${{SOCKET_PATH}}/stream_config.sock; error_log ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}}; @@ -216,7 +216,7 @@ server { > end -- database == "off" server { # ignore (and close }, to ignore content) - listen unix:${{PREFIX}}/stream_rpc.sock; + listen unix:${{SOCKET_PATH}}/stream_rpc.sock; error_log ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}}; content_by_lua_block { Kong.stream_api() @@ -225,7 +225,7 @@ server { # ignore (and close }, to ignore content) > end -- #stream_listeners > 0 server { - listen unix:${{PREFIX}}/stream_worker_events.sock; + listen unix:${{SOCKET_PATH}}/stream_worker_events.sock; error_log ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}}; access_log off; content_by_lua_block { diff --git a/kong/tools/stream_api.lua b/kong/tools/stream_api.lua index 1710487552be..ac8d7c09b77d 100644 --- a/kong/tools/stream_api.lua +++ b/kong/tools/stream_api.lua @@ -3,6 +3,7 @@ -- may changed or be removed in the future Kong releases once a better mechanism -- for inter subsystem communication in OpenResty became available. +local constants = require "kong.constants" local lpack = require "lua_pack" local kong = kong @@ -37,7 +38,9 @@ local MAX_DATA_LEN = 2^22 - 1 local HEADER_LEN = #st_pack(PACK_F, MAX_KEY_LEN, MAX_DATA_LEN) -local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/stream_rpc.sock" +-- this module may be loaded before `kong.configuration` is initialized +local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/" + .. constants.SOCKET_DIRECTORY .. "/stream_rpc.sock" local stream_api = {} diff --git a/spec/01-unit/01-db/11-declarative_lmdb_spec.lua b/spec/01-unit/01-db/11-declarative_lmdb_spec.lua index e1b2a79fa216..42756078b8ac 100644 --- a/spec/01-unit/01-db/11-declarative_lmdb_spec.lua +++ b/spec/01-unit/01-db/11-declarative_lmdb_spec.lua @@ -187,7 +187,7 @@ describe("#off preserve nulls", function() kong.configuration = kong_config kong.worker_events = kong.worker_events or kong.cache and kong.cache.worker_events or - assert(kong_global.init_worker_events()) + assert(kong_global.init_worker_events(kong.configuration)) kong.cluster_events = kong.cluster_events or kong.cache and kong.cache.cluster_events or assert(kong_global.init_cluster_events(kong.configuration, kong.db)) diff --git a/spec/01-unit/03-conf_loader_spec.lua b/spec/01-unit/03-conf_loader_spec.lua index 47c96492e443..604792c60476 100644 --- a/spec/01-unit/03-conf_loader_spec.lua +++ b/spec/01-unit/03-conf_loader_spec.lua @@ -2395,6 +2395,7 @@ describe("Configuration loader", function() local FIELDS = { -- CONF_BASIC prefix = true, + socket_path = true, vaults = true, database = true, lmdb_environment_path = true, diff --git a/spec/02-integration/02-cmd/02-start_stop_spec.lua b/spec/02-integration/02-cmd/02-start_stop_spec.lua index 15b61ad52551..31c55665d003 100644 --- a/spec/02-integration/02-cmd/02-start_stop_spec.lua +++ b/spec/02-integration/02-cmd/02-start_stop_spec.lua @@ -10,12 +10,14 @@ local read_file = helpers.file.read local PREFIX = helpers.test_conf.prefix +local SOCKET_PATH = helpers.test_conf.socket_path local TEST_CONF = helpers.test_conf local TEST_CONF_PATH = helpers.test_conf_path local function wait_until_healthy(prefix) prefix = prefix or PREFIX + local socket_path = prefix .. "/sockets" local cmd @@ -41,11 +43,11 @@ local function wait_until_healthy(prefix) local conf = assert(helpers.get_running_conf(prefix)) if conf.proxy_listen and conf.proxy_listen ~= "off" then - helpers.wait_for_file("socket", prefix .. "/worker_events.sock") + helpers.wait_for_file("socket", socket_path .. "/worker_events.sock") end if conf.stream_listen and conf.stream_listen ~= "off" then - helpers.wait_for_file("socket", prefix .. "/stream_worker_events.sock") + helpers.wait_for_file("socket", socket_path .. "/stream_worker_events.sock") end if conf.admin_listen and conf.admin_listen ~= "off" then @@ -1034,11 +1036,51 @@ describe("kong start/stop #" .. strategy, function() end) end) + describe("socket_path", function() + it("is created on demand by `kong prepare`", function() + local dir, cleanup = helpers.make_temp_dir() + finally(cleanup) + + local cmd = fmt("prepare -p %q", dir) + assert.truthy(kong_exec(cmd), "expected '" .. cmd .. "' to succeed") + assert.truthy(helpers.path.isdir(dir .. "/sockets"), + "expected '" .. dir .. "/sockets' directory to be created") + end) + + it("can be a user-created symlink", function() + local prefix, cleanup = helpers.make_temp_dir() + finally(cleanup) + + local socket_path + socket_path, cleanup = helpers.make_temp_dir() + finally(cleanup) + + assert.truthy(helpers.execute(fmt("ln -sf %q %q/sockets", socket_path, prefix)), + "failed to symlink socket path") + + local preserve_prefix = true + assert(helpers.start_kong({ + prefix = prefix, + database = "off", + nginx_conf = "spec/fixtures/custom_nginx.template", + }, nil, preserve_prefix)) + + finally(function() + helpers.stop_kong(prefix) + end) + + wait_until_healthy(prefix) + + assert.truthy(helpers.path.exists(socket_path .. "/worker_events.sock"), + "worker events socket was not created in the socket_path dir") + end) + end) + describe("dangling socket cleanup", function() local pidfile = TEST_CONF.nginx_pid -- the worker events socket is just one of many unix sockets we use - local event_sock = PREFIX .. "/worker_events.sock" + local event_sock = SOCKET_PATH .. "/worker_events.sock" local env = { prefix = PREFIX, @@ -1134,7 +1176,7 @@ describe("kong start/stop #" .. strategy, function() local _, stderr = assert_start() assert.matches("[warn] Found dangling unix sockets in the prefix directory", stderr, nil, true) - assert.matches(PREFIX, stderr, nil, true) + assert.matches(SOCKET_PATH, stderr, nil, true) assert.matches("removing unix socket", stderr) assert.matches(event_sock, stderr, nil, true) @@ -1175,6 +1217,7 @@ describe("kong start/stop #" .. strategy, function() it("works with resty.events when KONG_PREFIX is a relative path", function() local prefix = "relpath" + local socket_path = "relpath/sockets" finally(function() -- this test uses a non-default prefix, so it must manage @@ -1201,8 +1244,8 @@ describe("kong start/stop #" .. strategy, function() -- wait until everything is running wait_until_healthy(prefix) - assert.truthy(helpers.path.exists(prefix .. "/worker_events.sock")) - assert.truthy(helpers.path.exists(prefix .. "/stream_worker_events.sock")) + assert.truthy(helpers.path.exists(socket_path .. "/worker_events.sock")) + assert.truthy(helpers.path.exists(socket_path .. "/stream_worker_events.sock")) local log = prefix .. "/logs/error.log" assert.logfile(log).has.no.line("[error]", true, 0) diff --git a/spec/02-integration/03-db/14-dao_spec.lua b/spec/02-integration/03-db/14-dao_spec.lua index a8f20498d1c2..0b15bc49b4a6 100644 --- a/spec/02-integration/03-db/14-dao_spec.lua +++ b/spec/02-integration/03-db/14-dao_spec.lua @@ -85,7 +85,7 @@ for _, strategy in helpers.all_strategies() do local kong_global = require("kong.global") local kong = _G.kong - kong.worker_events = assert(kong_global.init_worker_events()) + kong.worker_events = assert(kong_global.init_worker_events(kong.configuration)) kong.cluster_events = assert(kong_global.init_cluster_events(kong.configuration, kong.db)) kong.cache = assert(kong_global.init_cache(kong.configuration, kong.cluster_events, kong.worker_events)) kong.core_cache = assert(kong_global.init_core_cache(kong.configuration, kong.cluster_events, kong.worker_events)) diff --git a/spec/02-integration/05-proxy/01-proxy_spec.lua b/spec/02-integration/05-proxy/01-proxy_spec.lua index 53f31a050f43..6192d3ab4e64 100644 --- a/spec/02-integration/05-proxy/01-proxy_spec.lua +++ b/spec/02-integration/05-proxy/01-proxy_spec.lua @@ -102,10 +102,10 @@ describe("#stream proxy interface listeners", function() stream_listen = "127.0.0.1:9011, 127.0.0.1:9012", })) - local stream_events_sock_path = "unix:" .. helpers.test_conf.prefix .. "/stream_worker_events.sock" + local stream_events_sock_path = "unix:" .. helpers.test_conf.socket_path .. "/stream_worker_events.sock" if helpers.test_conf.database == "off" then - local stream_config_sock_path = "unix:" .. helpers.test_conf.prefix .. "/stream_config.sock" + local stream_config_sock_path = "unix:" .. helpers.test_conf.socket_path .. "/stream_config.sock" assert.equals(3, count_server_blocks(helpers.test_conf.nginx_kong_stream_conf)) assert.same({ diff --git a/spec/02-integration/12-stream_api/01-stream_api_endpoint_spec.lua b/spec/02-integration/12-stream_api/01-stream_api_endpoint_spec.lua index 05385fd73978..a60c60c489f3 100644 --- a/spec/02-integration/12-stream_api/01-stream_api_endpoint_spec.lua +++ b/spec/02-integration/12-stream_api/01-stream_api_endpoint_spec.lua @@ -13,7 +13,7 @@ describe("Stream module API endpoint", function() plugins = "stream-api-echo", }) - socket_path = "unix:" .. helpers.get_running_conf().prefix .. "/stream_rpc.sock" + socket_path = "unix:" .. helpers.get_running_conf().socket_path .. "/stream_rpc.sock" end) lazy_teardown(function() diff --git a/spec/fixtures/template_inject/nginx_kong_test_custom_inject_stream.lua b/spec/fixtures/template_inject/nginx_kong_test_custom_inject_stream.lua index 7d43af7446c4..5306bd8803ef 100644 --- a/spec/fixtures/template_inject/nginx_kong_test_custom_inject_stream.lua +++ b/spec/fixtures/template_inject/nginx_kong_test_custom_inject_stream.lua @@ -33,7 +33,7 @@ include '*.stream_mock'; > if cluster_ssl_tunnel then server { - listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock; + listen unix:${{SOCKET_PATH}}/cluster_proxy_ssl_terminator.sock; proxy_pass ${{cluster_ssl_tunnel}}; proxy_ssl on; diff --git a/spec/helpers.lua b/spec/helpers.lua index bdcb04cf13ef..5fb537e8c6e4 100644 --- a/spec/helpers.lua +++ b/spec/helpers.lua @@ -545,7 +545,7 @@ end -- @param db the database object -- @return ml_cache instance local function get_cache(db) - local worker_events = assert(kong_global.init_worker_events()) + local worker_events = assert(kong_global.init_worker_events(conf)) local cluster_events = assert(kong_global.init_cluster_events(conf, db)) local cache = assert(kong_global.init_cache(conf, cluster_events, @@ -3843,10 +3843,13 @@ end -- @param preserve_dc ??? local function cleanup_kong(prefix, preserve_prefix, preserve_dc) -- remove socket files to ensure `pl.dir.rmtree()` ok - local socks = { "/worker_events.sock", "/stream_worker_events.sock", } - for _, name in ipairs(socks) do - local sock_file = (prefix or conf.prefix) .. name - os.remove(sock_file) + prefix = prefix or conf.prefix + local socket_path = pl_path.join(prefix, constants.SOCKET_DIRECTORY) + for child in lfs.dir(socket_path) do + if child:sub(-5) == ".sock" then + local path = pl_path.join(socket_path, child) + os.remove(path) + end end -- note: set env var "KONG_TEST_DONT_CLEAN" !! the "_TEST" will be dropped