refactor(nginx-templates): reduce redundancy

kong/cmd/reload.lua

@@ -36,7 +36,8 @@ local function execute(args)
     conf.declarative_config = nil
   end
 
-  assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, true))
+  assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, true,
+         args.nginx_conf_flags))
 
   _G.kong = kong_global.new()
   kong_global.init_pdk(_G.kong, conf)
@@ -62,9 +63,11 @@ and stop the old ones when they have finished processing
 current requests.
 
 Options:
- -c,--conf        (optional string) configuration file
- -p,--prefix      (optional string) prefix Kong is running at
- --nginx-conf     (optional string) custom Nginx configuration template
+ -c,--conf                 (optional string) configuration file
+ -p,--prefix               (optional string) prefix Kong is running at
+ --nginx-conf              (optional string) custom Nginx configuration template
+ --nginx-conf-flags        (optional string) flags that can be used to control
+                                             how Nginx configuration templates are rendered
 ]]
 
 

kong/cmd/restart.lua

@@ -43,12 +43,14 @@ This command is equivalent to doing both 'kong stop' and
 'kong start'.
 
 Options:
- -c,--conf        (optional string)   configuration file
- -p,--prefix      (optional string)   prefix at which Kong should be running
- --nginx-conf     (optional string)   custom Nginx configuration template
- --run-migrations (optional boolean)  optionally run migrations on the DB
- --db-timeout     (default 60)
- --lock-timeout   (default 60)
+ -c,--conf                 (optional string)   configuration file
+ -p,--prefix               (optional string)   prefix at which Kong should be running
+ --nginx-conf              (optional string)   custom Nginx configuration template
+ --run-migrations          (optional boolean)  optionally run migrations on the DB
+ --db-timeout              (default 60)
+ --lock-timeout            (default 60)
+ --nginx-conf-flags        (optional string)   flags that can be used to control
+                                               how Nginx configuration templates are rendered
 ]]
 
 return {

kong/cmd/start.lua

@@ -56,7 +56,8 @@ local function execute(args)
   assert(not kill.is_running(conf.nginx_pid),
          "Kong is already running in " .. conf.prefix)
 
-  assert(prefix_handler.prepare_prefix(conf, args.nginx_conf))
+  assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, nil,
+         args.nginx_conf_flags))
 
   cleanup_dangling_unix_sockets(conf.prefix)
 
@@ -117,20 +118,23 @@ Start Kong (Nginx and other configured services) in the configured
 prefix directory.
 
 Options:
- -c,--conf        (optional string)   Configuration file.
+ -c,--conf                 (optional string)   Configuration file.
 
- -p,--prefix      (optional string)   Override prefix directory.
+ -p,--prefix               (optional string)   Override prefix directory.
 
- --nginx-conf     (optional string)   Custom Nginx configuration template.
+ --nginx-conf              (optional string)   Custom Nginx configuration template.
 
- --run-migrations (optional boolean)  Run migrations before starting.
+ --run-migrations          (optional boolean)  Run migrations before starting.
 
- --db-timeout     (default 60)        Timeout, in seconds, for all database
-                                      operations.
+ --db-timeout              (default 60)        Timeout, in seconds, for all database
+                                               operations.
 
- --lock-timeout   (default 60)        When --run-migrations is enabled, timeout,
-                                      in seconds, for nodes waiting on the
-                                      leader node to finish running migrations.
+ --lock-timeout            (default 60)        When --run-migrations is enabled, timeout,
+                                               in seconds, for nodes waiting on the
+                                               leader node to finish running migrations.
+
+ --nginx-conf-flags        (optional string)   Flags that can be used to control
+                                               how Nginx configuration templates are rendered
 ]]
 
 return {

kong/cmd/utils/prefix_handler.lua

@@ -235,7 +235,7 @@ local function get_ulimit()
   end
 end
 
-local function compile_conf(kong_config, conf_template)
+local function compile_conf(kong_config, conf_template, template_env_inject)
   -- computed config properties for templating
   local compile_env = {
     _escape = ">",
@@ -247,6 +247,8 @@ local function compile_conf(kong_config, conf_template)
     }
   }
 
+  compile_env = pl_tablex.merge(compile_env, template_env_inject or {}, true)
+
   do
     local worker_rlimit_nofile_auto
     if kong_config.nginx_main_directives then
@@ -388,16 +390,16 @@ local function write_process_secrets_file(path, data)
   return true
 end
 
-local function compile_kong_conf(kong_config)
-  return compile_conf(kong_config, kong_nginx_template)
+local function compile_kong_conf(kong_config, template_env_inject)
+  return compile_conf(kong_config, kong_nginx_template, template_env_inject)
 end
 
 local function compile_kong_gui_include_conf(kong_config)
   return compile_conf(kong_config, kong_nginx_gui_include_template)
 end
 
-local function compile_kong_stream_conf(kong_config)
-  return compile_conf(kong_config, kong_nginx_stream_template)
+local function compile_kong_stream_conf(kong_config, template_env_inject)
+  return compile_conf(kong_config, kong_nginx_stream_template, template_env_inject)
 end
 
 local function compile_nginx_conf(kong_config, template)
@@ -436,7 +438,11 @@ local function compile_nginx_stream_inject_conf(kong_config)
   return compile_conf(kong_config, nginx_stream_inject_template)
 end
 
-local function prepare_prefix(kong_config, nginx_custom_template_path, skip_write, write_process_secrets)
+local function compile_kong_test_inject_conf(kong_config, template, template_env)
+  return compile_conf(kong_config, template, template_env)
+end
+
+local function prepare_prefix(kong_config, nginx_custom_template_path, skip_write, write_process_secrets, nginx_conf_flags)
   log.verbose("preparing nginx prefix directory at %s", kong_config.prefix)
 
   if not exists(kong_config.prefix) then
@@ -678,7 +684,12 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     gen_default_dhparams(kong_config)
   end
 
-  -- write NGINX conf
+  local template_env = {}
+  nginx_conf_flags = nginx_conf_flags and pl_stringx.split(nginx_conf_flags, ",") or {}
+  for _, flag in ipairs(nginx_conf_flags) do
+    template_env[flag] = true
+  end
+
   local nginx_conf, err = compile_nginx_conf(kong_config, nginx_template)
   if not nginx_conf then
     return nil, err
@@ -693,14 +704,14 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
   pl_file.write(kong_config.nginx_kong_gui_include_conf, nginx_kong_gui_include_conf)
 
   -- write Kong's HTTP NGINX conf
-  local nginx_kong_conf, err = compile_kong_conf(kong_config)
+  local nginx_kong_conf, err = compile_kong_conf(kong_config, template_env)
   if not nginx_kong_conf then
     return nil, err
   end
   pl_file.write(kong_config.nginx_kong_conf, nginx_kong_conf)
 
   -- write Kong's stream NGINX conf
-  local nginx_kong_stream_conf, err = compile_kong_stream_conf(kong_config)
+  local nginx_kong_stream_conf, err = compile_kong_stream_conf(kong_config, template_env)
   if not nginx_kong_stream_conf then
     return nil, err
   end
@@ -727,6 +738,29 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
   end
   pl_file.write(kong_config.nginx_kong_stream_inject_conf, nginx_stream_inject_conf)
 
+  -- write Kong's test injected configuration files (*.test.conf)
+  -- these are included in the Kong's HTTP NGINX conf by the test template
+  local test_template_inj_path = "spec/fixtures/template_inject/"
+  if pl_path.isdir(test_template_inj_path) then
+    for _, file in ipairs(pl_dir.getfiles(test_template_inj_path, "*.lua")) do
+      local t_path = pl_path.splitext(file)
+      local t_module = string.gsub(t_path, "/", ".")
+      local nginx_kong_test_inject_conf, err = compile_kong_test_inject_conf(
+        kong_config,
+        require(t_module),
+        template_env
+      )
+
+      if not nginx_kong_test_inject_conf then
+        return nil, err
+      end
+
+      local t_name = pl_path.basename(t_path)
+      local output_path = kong_config.prefix .. "/" .. t_name .. ".test.conf"
+      pl_file.write(output_path, nginx_kong_test_inject_conf)
+    end
+  end
+
   -- testing written NGINX conf
   local ok, err = nginx_signals.check_conf(kong_config)
   if not ok then

kong/templates/nginx_kong.lua

@@ -34,6 +34,10 @@ $(el.name) $(el.value);
 > end
 
 init_by_lua_block {
+> if test and coverage then
+    require 'luacov'
+    jit.off()
+> end -- test and coverage
     Kong = require 'kong'
     Kong.init()
 }

kong/templates/nginx_kong_stream.lua

@@ -34,6 +34,10 @@ $(el.name) $(el.value);
 > end
 
 init_by_lua_block {
+> if test and coverage then
+    require 'luacov'
+    jit.off()
+> end -- test and coverage
     -- shared dictionaries conflict between stream/http modules. use a prefix.
     local shared = ngx.shared
     local stream_shdict_prefix = "stream_"

spec/01-unit/04-prefix_handler_spec.lua

@@ -1333,7 +1333,8 @@ describe("NGINX conf compiler", function()
         local contents = helpers.file.read(tmp_config.nginx_conf)
         assert.matches("# This is a custom nginx configuration template for Kong specs", contents, nil, true)
         assert.matches("daemon%s+on;", contents)
-        assert.matches("listen%s+0%.0%.0%.0:9000;", contents)
+        local contents_kong_conf = helpers.file.read(tmp_config.nginx_kong_conf)
+        assert.matches("listen%s+0%.0%.0%.0:9000;", contents_kong_conf)
       end)
       it("errors on non-existing file", function()
         local ok, err = prefix_handler.prepare_prefix(tmp_config, "spec/fixtures/inexistent.template")

spec/02-integration/02-cmd/02-start_stop_spec.lua

@@ -1,5 +1,6 @@
 local helpers   = require "spec.helpers"
 local constants = require "kong.constants"
+local pl_file   = require("pl.file")
 
 local cjson = require "cjson"
 
@@ -587,25 +588,46 @@ describe("kong start/stop #" .. strategy, function()
     end)
 
     it("ensures the required shared dictionaries are defined", function()
-      local templ_fixture     = "spec/fixtures/custom_nginx.template"
-      local new_templ_fixture = "spec/fixtures/custom_nginx.template.tmp"
+      local tmp_nginx_config = "spec/fixtures/nginx_conf.tmp"
+      local prefix_handler = require "kong.cmd.utils.prefix_handler"
+      local conf_loader = require "kong.conf_loader"
 
-      finally(function()
-        os.remove(new_templ_fixture)
-      end)
+      local nginx_conf = assert(conf_loader(helpers.test_conf_path, {
+        prefix = "servroot_tmp",
+      }))
+      assert(prefix_handler.prepare_prefix(nginx_conf))
+      assert.truthy(helpers.path.exists(nginx_conf.nginx_conf))
+      local kong_nginx_conf = assert(prefix_handler.compile_kong_conf(nginx_conf))
 
       for _, dict in ipairs(constants.DICTS) do
         -- remove shared dictionary entry
-        assert(os.execute(fmt("sed '/lua_shared_dict %s .*;/d' %s > %s",
-                              dict, templ_fixture, new_templ_fixture)))
+        local http_cfg = string.gsub(kong_nginx_conf, "lua_shared_dict%s" .. dict .. "%s.-\n", "")
+        local conf = [[pid pids/nginx.pid;
+          error_log logs/error.log debug;
+          daemon on;
+          worker_processes 1;
+          events {
+            multi_accept off;
+          }
+          http {
+            ]]
+            .. http_cfg ..
+            [[
+          }
+        ]]
 
-        local ok, err = helpers.start_kong({ nginx_conf = new_templ_fixture })
+        pl_file.write(tmp_nginx_config, conf)
+        local ok, err = helpers.start_kong({ nginx_conf = tmp_nginx_config })
         assert.falsy(ok)
         assert.matches(
           "missing shared dict '" .. dict .. "' in Nginx configuration, "    ..
           "are you using a custom template? Make sure the 'lua_shared_dict " ..
           dict .. " [SIZE];' directive is defined.", err, nil, true)
       end
+
+      finally(function()
+        os.remove(tmp_nginx_config)
+      end)
     end)
 
     if strategy == "off" then

spec/02-integration/02-cmd/03-reload_spec.lua

@@ -740,7 +740,7 @@ describe("Admin GUI config", function ()
 
     client:close()
 
-    assert(helpers.reload_kong("off", "reload --conf " .. helpers.test_conf_path .. " --nginx-conf spec/fixtures/default_nginx.template", {
+    assert(helpers.reload_kong("off", "reload --conf " .. helpers.test_conf_path, {
       database = "off",
       admin_gui_listen = "127.0.0.1:9012",
       admin_gui_url = "http://test2.example.com",

spec/03-plugins/06-statsd/01-log_spec.lua

@@ -1,5 +1,7 @@
 local helpers       = require "spec.helpers"
 local pl_file       = require "pl.file"
+local pl_dir        = require "pl.dir"
+local pl_path       = require "pl.path"
 
 local get_hostname = require("kong.pdk.node").new().get_hostname
 
@@ -17,16 +19,33 @@ local uuid_pattern = "%x%x%x%x%x%x%x%x%-%x%x%x%x%-4%x%x%x%-%x%x%x%x%-%x%x%x%x%x%
 local workspace_name_pattern = "default"
 
 
-local function get_shdicts()
+local function count_shdicts(conf_file)
   local prefix = helpers.test_conf.prefix
-  local ngxconf = helpers.utils.readfile(prefix .. "/nginx.conf")
-  local pattern = "\n%s*lua_shared_dict%s+(.-)[%s;\n]"
-  local shdicts = {}
-  for dict_name in ngxconf:gmatch(pattern) do
-    table.insert(shdicts, dict_name)
-    --print(#shdicts, "-", dict_name)
+  local counter = 0
+
+  -- count in matched `*` files
+  if conf_file:find("*") then
+    for _, file in ipairs(pl_dir.getallfiles(prefix, conf_file)) do
+      local basename = pl_path.basename(file)
+      counter = counter + count_shdicts(basename)
+    end
+    return counter
+  end
+
+  -- count in the current file
+  local ngx_conf = helpers.utils.readfile(prefix .. "/" .. conf_file)
+  local dict_ptrn = "%s*lua_shared_dict%s+(.-)[%s;\n]"
+  for _ in ngx_conf:gmatch(dict_ptrn) do
+    counter = counter + 1
   end
-  return shdicts
+
+  -- count in other included files
+  local include_ptrn = "%s*include%s+'(.-%.conf)'[;\n]"
+  for include_file in ngx_conf:gmatch(include_ptrn) do
+    counter = counter + count_shdicts(include_file)
+  end
+
+  return counter
 end
 
 
@@ -862,7 +881,7 @@ for _, strategy in helpers.each_strategy() do
 
       proxy_client = helpers.proxy_client()
       proxy_client_grpc = helpers.proxy_client_grpc()
-      shdict_count = #get_shdicts()
+      shdict_count = count_shdicts("nginx.conf")
     end)
 
     lazy_teardown(function()
@@ -894,7 +913,7 @@ for _, strategy in helpers.each_strategy() do
 
         proxy_client = helpers.proxy_client()
         proxy_client_grpc = helpers.proxy_client_grpc()
-        shdict_count = #get_shdicts()
+        shdict_count = count_shdicts("nginx.conf")
       end)
 
       it("logs over UDP with default metrics", function()
@@ -2054,7 +2073,7 @@ for _, strategy in helpers.each_strategy() do
           nginx_conf = "spec/fixtures/custom_nginx.template",
         }))
 
-        shdict_count = #get_shdicts()
+        shdict_count = count_shdicts("nginx.conf")
 
         local metrics_count = expected_metrics_count(8)
         local thread = helpers.udp_server(UDP_PORT, metrics_count, 2)
@@ -2312,7 +2331,7 @@ for _, strategy in helpers.each_strategy() do
       }))
 
       proxy_client = helpers.proxy_client()
-      shdict_count = #get_shdicts()
+      shdict_count = count_shdicts("nginx.conf")
     end)
 
     lazy_teardown(function()