fix(core): relocate unix sockets to a subdirectory (#13409)

build/dockerfiles/entrypoint.sh

@@ -46,11 +46,12 @@ if [[ "$1" == "kong" ]]; then
 
     # remove all dangling sockets in $PREFIX dir before starting Kong
     LOGGED_SOCKET_WARNING=0
-    for localfile in "$PREFIX"/*; do
+    socket_path=$PREFIX/sockets
+    for localfile in "$socket_path"/*; do
       if [ -S "$localfile" ]; then
         if (( LOGGED_SOCKET_WARNING == 0 )); then
           printf >&2 'WARN: found dangling unix sockets in the prefix directory '
-          printf >&2 '(%q) ' "$PREFIX"
+          printf >&2 '(%q) ' "$socket_path"
           printf >&2 'while preparing to start Kong. This may be a sign that Kong '
           printf >&2 'was previously shut down uncleanly or is in an unknown state '
           printf >&2 'and could require further investigation.\n'

changelog/unreleased/kong/move-sockets-to-subdir.yml

@@ -0,0 +1,3 @@
+message: Moved internal Unix sockets to a subdirectory (`sockets`) of the Kong prefix.
+type: bugfix
+scope: Core

kong/clustering/utils.lua

@@ -25,8 +25,8 @@ local _log_prefix = "[clustering] "
 
 local KONG_VERSION = kong.version
 
-local prefix = kong.configuration.prefix or require("pl.path").abspath(ngx.config.prefix())
-local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock", prefix)
+local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock",
+                                              kong.configuration.socket_path)
 
 local _M = {}
 

kong/cmd/start.lua

@@ -13,11 +13,11 @@ local function is_socket(path)
   return lfs.attributes(path, "mode") == "socket"
 end
 
-local function cleanup_dangling_unix_sockets(prefix)
+local function cleanup_dangling_unix_sockets(socket_path)
   local found = {}
 
-  for child in lfs.dir(prefix) do
-    local path = prefix .. "/" .. child
+  for child in lfs.dir(socket_path) do
+    local path = socket_path .. "/" .. child
     if is_socket(path) then
       table.insert(found, path)
     end
@@ -31,7 +31,7 @@ local function cleanup_dangling_unix_sockets(prefix)
            "preparing to start Kong. This may be a sign that Kong was " ..
            "previously shut down uncleanly or is in an unknown state and " ..
            "could require further investigation.",
-           prefix)
+           socket_path)
 
   log.warn("Attempting to remove dangling sockets before starting Kong...")
 
@@ -59,7 +59,7 @@ local function execute(args)
   assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, nil,
          args.nginx_conf_flags))
 
-  cleanup_dangling_unix_sockets(conf.prefix)
+  cleanup_dangling_unix_sockets(conf.socket_path)
 
   _G.kong = kong_global.new()
   kong_global.init_pdk(_G.kong, conf)

kong/cmd/utils/prefix_handler.lua

@@ -481,6 +481,13 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     return nil, kong_config.prefix .. " is not a directory"
   end
 
+  if not exists(kong_config.socket_path) then
+    local ok, err = makepath(kong_config.socket_path)
+    if not ok then
+      return nil, err
+    end
+  end
+
   -- create directories in prefix
   for _, dir in ipairs {"logs", "pids"} do
     local ok, err = makepath(join(kong_config.prefix, dir))

kong/conf_loader/init.lua

@@ -17,6 +17,7 @@ local pl_path = require "pl.path"
 local tablex = require "pl.tablex"
 local log = require "kong.cmd.utils.log"
 local env = require "kong.cmd.utils.env"
+local constants = require "kong.constants"
 
 
 local cycle_aware_deep_copy = require("kong.tools.table").cycle_aware_deep_copy
@@ -482,6 +483,10 @@ local function load(path, custom_conf, opts)
   -- load absolute paths
   conf.prefix = abspath(conf.prefix)
 
+  -- The socket path is where we store listening unix sockets for IPC and private APIs.
+  -- It is derived from the prefix and is NOT intended to be user-configurable
+  conf.socket_path = pl_path.join(conf.prefix, constants.SOCKET_DIRECTORY)
+
   if conf.lua_ssl_trusted_certificate
      and #conf.lua_ssl_trusted_certificate > 0 then
 

kong/constants.lua

@@ -280,6 +280,8 @@ local constants = {
       service = "upstream",
     }
   },
+
+  SOCKET_DIRECTORY = "sockets",
 }
 
 for _, v in ipairs(constants.CLUSTERING_SYNC_STATUS) do

kong/global.lua

@@ -168,28 +168,20 @@ function _GLOBAL.init_pdk(self, kong_config)
 end
 
 
-function _GLOBAL.init_worker_events()
+function _GLOBAL.init_worker_events(kong_config)
   -- Note: worker_events will not work correctly if required at the top of the file.
   --       It must be required right here, inside the init function
   local worker_events
   local opts
 
-  local configuration = kong.configuration
-
-  -- `kong.configuration.prefix` is already normalized to an absolute path,
-  -- but `ngx.config.prefix()` is not
-  local prefix = configuration and
-                 configuration.prefix or
-                 require("pl.path").abspath(ngx.config.prefix())
-
+  local socket_path = kong_config.socket_path
   local sock = ngx.config.subsystem == "stream" and
                "stream_worker_events.sock" or
                "worker_events.sock"
 
-  local listening = "unix:" .. prefix .. "/" .. sock
+  local listening = "unix:" .. socket_path .. "/" .. sock
 
-  local max_payload_len = configuration and
-                          configuration.worker_events_max_payload
+  local max_payload_len = kong_config.worker_events_max_payload
 
   if max_payload_len and max_payload_len > 65535 then   -- default is 64KB
     ngx.log(ngx.WARN,
@@ -203,9 +195,9 @@ function _GLOBAL.init_worker_events()
     listening = listening,  -- unix socket for broker listening
     max_queue_len = 1024 * 50,  -- max queue len for events buffering
     max_payload_len = max_payload_len,  -- max payload size in bytes
-    enable_privileged_agent = configuration and configuration.dedicated_config_processing
-                                            and configuration.role == "data_plane"
-                                             or false
+    enable_privileged_agent = kong_config.dedicated_config_processing
+                          and kong_config.role == "data_plane"
+                           or false,
   }
 
   worker_events = require "resty.events.compat"

kong/init.lua

@@ -837,7 +837,7 @@ function Kong.init_worker()
 
   schema_state = nil
 
-  local worker_events, err = kong_global.init_worker_events()
+  local worker_events, err = kong_global.init_worker_events(kong.configuration)
   if not worker_events then
     stash_init_worker_error("failed to instantiate 'kong.worker_events' " ..
                             "module: " .. err)

kong/runloop/events.lua

@@ -507,12 +507,18 @@ local stream_reconfigure_listener
 do
   local buffer = require "string.buffer"
 
-  -- `kong.configuration.prefix` is already normalized to an absolute path,
-  -- but `ngx.config.prefix()` is not
-  local PREFIX = kong and kong.configuration and
-                 kong.configuration.prefix or
-                 require("pl.path").abspath(ngx.config.prefix())
-  local STREAM_CONFIG_SOCK = "unix:" .. PREFIX .. "/stream_config.sock"
+  -- this module may be loaded before `kong.configuration` is initialized
+  local socket_path = kong and kong.configuration
+                      and kong.configuration.socket_path
+
+  if not socket_path then
+    -- `kong.configuration.socket_path` is already normalized to an absolute
+    -- path, but `ngx.config.prefix()` is not
+    socket_path = require("pl.path").abspath(ngx.config.prefix() .. "/"
+                                             .. constants.SOCKET_DIRECTORY)
+  end
+
+  local STREAM_CONFIG_SOCK = "unix:" .. socket_path .. "/stream_config.sock"
   local IS_HTTP_SUBSYSTEM  = ngx.config.subsystem == "http"
 
   local function broadcast_reconfigure_event(data)

kong/runloop/handler.lua

@@ -896,11 +896,9 @@ return {
 
   init_worker = {
     before = function()
-      -- TODO: PR #9337 may affect the following line
-      local prefix = kong.configuration.prefix or ngx.config.prefix()
-
-      STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", prefix)
-      STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", prefix)
+      local socket_path = kong.configuration.socket_path
+      STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", socket_path)
+      STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", socket_path)
 
       log_level.init_worker()
 

kong/templates/nginx.lua

@@ -83,7 +83,7 @@ stream {
 
 > if cluster_ssl_tunnel then
     server {
-        listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock;
+        listen unix:${{SOCKET_PATH}}/cluster_proxy_ssl_terminator.sock;
 
         proxy_pass ${{cluster_ssl_tunnel}};
         proxy_ssl on;

kong/templates/nginx_kong.lua

@@ -592,7 +592,7 @@ server {
 server {
     charset UTF-8;
     server_name kong_worker_events;
-    listen unix:${{PREFIX}}/worker_events.sock;
+    listen unix:${{SOCKET_PATH}}/worker_events.sock;
     access_log off;
     location / {
         content_by_lua_block {

kong/templates/nginx_kong_stream.lua

@@ -94,7 +94,7 @@ server {
 > end
 
 > if stream_proxy_ssl_enabled then
-    listen unix:${{PREFIX}}/stream_tls_terminate.sock ssl proxy_protocol;
+    listen unix:${{SOCKET_PATH}}/stream_tls_terminate.sock ssl proxy_protocol;
 > end
 
     access_log ${{PROXY_STREAM_ACCESS_LOG}};
@@ -175,7 +175,7 @@ server {
 }
 
 server {
-    listen unix:${{PREFIX}}/stream_tls_passthrough.sock proxy_protocol;
+    listen unix:${{SOCKET_PATH}}/stream_tls_passthrough.sock proxy_protocol;
 
     access_log ${{PROXY_STREAM_ACCESS_LOG}};
     error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}};
@@ -205,7 +205,7 @@ server {
 
 > if database == "off" then
 server {
-    listen unix:${{PREFIX}}/stream_config.sock;
+    listen unix:${{SOCKET_PATH}}/stream_config.sock;
 
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
 
@@ -216,7 +216,7 @@ server {
 > end -- database == "off"
 
 server {        # ignore (and close }, to ignore content)
-    listen unix:${{PREFIX}}/stream_rpc.sock;
+    listen unix:${{SOCKET_PATH}}/stream_rpc.sock;
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
     content_by_lua_block {
         Kong.stream_api()
@@ -225,7 +225,7 @@ server {        # ignore (and close }, to ignore content)
 > end -- #stream_listeners > 0
 
 server {
-    listen unix:${{PREFIX}}/stream_worker_events.sock;
+    listen unix:${{SOCKET_PATH}}/stream_worker_events.sock;
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
     access_log off;
     content_by_lua_block {

kong/tools/stream_api.lua

@@ -3,6 +3,7 @@
 -- may changed or be removed in the future Kong releases once a better mechanism
 -- for inter subsystem communication in OpenResty became available.
 
+local constants = require "kong.constants"
 local lpack = require "lua_pack"
 
 local kong = kong
@@ -37,7 +38,9 @@ local MAX_DATA_LEN = 2^22 - 1
 
 local HEADER_LEN = #st_pack(PACK_F, MAX_KEY_LEN, MAX_DATA_LEN)
 
-local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/stream_rpc.sock"
+-- this module may be loaded before `kong.configuration` is initialized
+local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/"
+                    .. constants.SOCKET_DIRECTORY .. "/stream_rpc.sock"
 
 local stream_api = {}
 

spec/01-unit/01-db/11-declarative_lmdb_spec.lua

@@ -187,7 +187,7 @@ describe("#off preserve nulls", function()
     kong.configuration = kong_config
     kong.worker_events = kong.worker_events or
                          kong.cache and kong.cache.worker_events or
-                         assert(kong_global.init_worker_events())
+                         assert(kong_global.init_worker_events(kong.configuration))
     kong.cluster_events = kong.cluster_events or
                           kong.cache and kong.cache.cluster_events or
                           assert(kong_global.init_cluster_events(kong.configuration, kong.db))

spec/01-unit/03-conf_loader_spec.lua

@@ -2395,6 +2395,7 @@ describe("Configuration loader", function()
       local FIELDS = {
         -- CONF_BASIC
         prefix = true,
+        socket_path = true,
         vaults = true,
         database = true,
         lmdb_environment_path = true,