feat(dns): use new resolver_* directives for new DNS client (#13427)

This avoids confusion with old DNS client config as not all of them works in the new client.

KAG-4994

changelog/unreleased/kong/refactor_dns_client.yml

@@ -2,8 +2,6 @@ message: >
   Starting from this version, a new DNS client library has been implemented and added into Kong. The new DNS client library has the following changes
   - Introduced global caching for DNS records across workers, significantly reducing the query load on DNS servers.
   - Introduced observable statistics for the new DNS client, and a new Admin API `/status/dns` to retrieve them.
-  - Deprecated the `dns_no_sync` option. Multiple DNS queries for the same name will always be synchronized (even across workers). This remains functional with the legacy DNS client library.
-  - Deprecated the `dns_not_found_ttl` option. It uses the `dns_error_ttl` option for all error responses. This option remains functional with the legacy DNS client library.
-  - Deprecated the `dns_order` option. By default, SRV, A, and AAAA are supported. Only names in the SRV format (`_service._proto.name`) enable resolving of DNS SRV records.
+  - Simplified the logic and make it more standardized
 type: feature
 scope: Core

kong.conf.default

@@ -1532,6 +1532,107 @@
                                  # same name/type will be synchronized to a
                                  # single query.
 
+#------------------------------------------------------------------------------
+# New DNS RESOLVER
+#------------------------------------------------------------------------------
+
+# This DNS resolver introduces global caching for DNS records across workers,
+# significantly reducing the query load on DNS servers.
+#
+# It provides observable statistics, you can retrieve them through the Admin API
+# `/status/dns`.
+
+#legacy_dns_client = off         # Disable the new DNS resolver, using the
+                                 # original DNS resolver. See above `dns_xxx`
+                                 # options for the original DNS resolver.
+
+#resolver_address = <name servers parsed from resolv.conf>
+                                 # Comma-separated list of nameservers, each
+                                 # entry in `ip[:port]` format to be used by
+                                 # Kong. If not specified, the nameservers in
+                                 # the local `resolv.conf` file will be used.
+                                 # Port defaults to 53 if omitted. Accepts
+                                 # both IPv4 and IPv6 addresses.
+                                 #
+                                 # Examples:
+                                 #
+                                 # ```
+                                 # resolver_address = 8.8.8.8
+                                 # resolver_address = 8.8.8.8, [::1]
+                                 # resolver_address = 8.8.8.8:53, [::1]:53
+                                 # ```
+
+#resolver_hosts_file = /etc/hosts
+                                 # The hosts file to use. This file is read
+                                 # once and its content is static in memory.
+                                 # To read the file again after modifying it,
+                                 # Kong must be reloaded.
+
+#resolver_family = A,SRV         # The supported query types.
+                                 #
+                                 # For a domain name, Kong will only query
+                                 # either IP addresses (A or AAAA) or SRV
+                                 # records, but not both.
+                                 #
+                                 # It will query SRV records only when the
+                                 # domain matches the
+                                 # "_<proto>._<service>.<name>" format, for
+                                 # example, "_ldap._tcp.example.com".
+                                 #
+                                 # For IP addresses (A or AAAA) resolution, it
+                                 # first attempts IPv4 (A) and then queries
+                                 # IPv6 (AAAA).
+
+#resolver_valid_ttl = <TTL from responses>
+                                 # By default, DNS records are cached using
+                                 # the TTL value of a response. This optional
+                                 # parameter (in seconds) allows overriding it.
+
+#resolver_error_ttl = 1          # TTL in seconds for error responses and empty
+                                 # responses.
+
+#resolver_stale_ttl = 3600       # Defines, in seconds, how long a record will
+                                 # remain in cache past its TTL. This value
+                                 # will be used while the new DNS record is
+                                 # fetched in the background.
+                                 #
+                                 # Stale data will be used from expiry of a
+                                 # record until either the refresh query
+                                 # completes, or the `resolver_stale_ttl` number
+                                 # of seconds have passed.
+                                 #
+                                 # This configuration enables Kong to be more
+                                 # resilient during the DNS server downtime.
+
+#resolver_lru_cache_size = 10000 # The DNS client uses a two-layer cache system:
+                                 # L1 - worker-level LRU Lua VM cache
+                                 # L2 - across-workers shared memory cache
+                                 #
+                                 # This value specifies the maximum allowed
+                                 # number of DNS responses stored in the L1 LRU
+                                 # lua VM cache.
+                                 #
+                                 # A single name query can easily take up 1~10
+                                 # slots, depending on attempted query types and
+                                 # extended domains from /etc/resolv.conf
+                                 # options `domain` or `search`.
+
+#resolver_mem_cache_size = 5m    # This value specifies the size of the L2
+                                 # shared memory cache for DNS responses,
+                                 # `kong_dns_cache`.
+                                 #
+                                 # Accepted units are `k` and `m`, with a
+                                 # minimum recommended value of a few MBs.
+                                 #
+                                 # 5MB shared memory size could store
+                                 # ~20000 DNS responeses with single A record or
+                                 # ~10000 DNS responeses with 2~3 A records.
+                                 #
+                                 # 10MB shared memory size could store
+                                 # ~40000 DNS responeses with single A record or
+                                 # ~20000 DNS responeses with 2~3 A records.
+
+
 #------------------------------------------------------------------------------
 # VAULTS
 #------------------------------------------------------------------------------

kong/conf_loader/constants.lua

@@ -370,7 +370,17 @@ local CONF_PARSERS = {
   dns_not_found_ttl = { typ = "number" },
   dns_error_ttl = { typ = "number" },
   dns_no_sync = { typ = "boolean" },
+
   legacy_dns_client = { typ = "boolean" },
+
+  resolver_address = { typ = "array" },
+  resolver_hosts_file = { typ = "string" },
+  resolver_family = { typ = "array" },
+  resolver_valid_ttl = { typ = "number" },
+  resolver_stale_ttl = { typ = "number" },
+  resolver_error_ttl = { typ = "number" },
+  resolver_lru_cache_size = { typ = "number" },
+
   privileged_worker = {
     typ = "boolean",
     deprecated = {

kong/conf_loader/parse.lua

@@ -529,6 +529,37 @@ local function check_and_parse(conf, opts)
     end
   end
 
+  --- new dns client
+
+  if conf.resolver_address then
+    for _, server in ipairs(conf.resolver_address) do
+      local dns = normalize_ip(server)
+
+      if not dns or dns.type == "name" then
+        errors[#errors + 1] = "resolver_address must be a comma separated list " ..
+                              "in the form of IPv4/6 or IPv4/6:port, got '"  ..
+                              server .. "'"
+      end
+    end
+  end
+
+  if conf.resolver_hosts_file then
+    if not pl_path.isfile(conf.resolver_hosts_file) then
+      errors[#errors + 1] = "resolver_hosts_file: file does not exist"
+    end
+  end
+
+  if conf.resolver_family then
+    local allowed = { A = true, AAAA = true, SRV = true }
+
+    for _, name in ipairs(conf.resolver_family) do
+      if not allowed[upper(name)] then
+        errors[#errors + 1] = fmt("resolver_family: invalid entry '%s'",
+                                  tostring(name))
+      end
+    end
+  end
+
   if not conf.lua_package_cpath then
     conf.lua_package_cpath = ""
   end

kong/dns/client.lua

@@ -617,13 +617,6 @@ local dns_client
 function _M.init(opts)
   log(DEBUG, PREFIX, "(re)configuring dns client")
 
-  if opts then
-    opts.valid_ttl = opts.valid_ttl or opts.validTtl
-    opts.error_ttl = opts.error_ttl or opts.badTtl
-    opts.stale_ttl = opts.stale_ttl or opts.staleTtl
-    opts.cache_size = opts.cache_size or opts.cacheSize
-  end
-
   local client, err = _M.new(opts)
   if not client then
     return nil, err

kong/templates/kong_defaults.lua

@@ -169,8 +169,18 @@ dns_cache_size = 10000
 dns_not_found_ttl = 30
 dns_error_ttl = 1
 dns_no_sync = off
+
 legacy_dns_client = off
 
+resolver_address = NONE
+resolver_hosts_file = /etc/hosts
+resolver_family = A,SRV
+resolver_valid_ttl = NONE
+resolver_stale_ttl = 3600
+resolver_lru_cache_size = 10000
+resolver_mem_cache_size = 5m
+resolver_error_ttl = 1
+
 dedicated_config_processing = on
 worker_consistency = eventual
 worker_state_update_frequency = 5

kong/templates/nginx_kong.lua

@@ -24,7 +24,7 @@ lua_shared_dict kong_db_cache_miss          12m;
 lua_shared_dict kong_secrets                5m;
 
 > if not legacy_dns_client then
-lua_shared_dict kong_dns_cache              5m;
+lua_shared_dict kong_dns_cache              ${{RESOLVER_MEM_CACHE_SIZE}};
 > end
 
 underscores_in_headers on;

kong/tools/dns.lua

@@ -37,6 +37,30 @@ local setup_client = function(conf)
     noSynchronisation = conf.dns_no_sync,
   }
 
+  -- new dns client
+  if ngx.shared.kong_dns_cache and not _G.busted_legacy_dns_client then
+
+    servers = {}
+
+    if conf.resolver_address then
+      for i, server in ipairs(conf.resolver_address) do
+        local s = normalize_ip(server)
+        servers[i] = { s.host, s.port or 53 }   -- inserting port if omitted
+      end
+    end
+
+    opts = {
+      nameservers = servers,
+      hosts = conf.resolver_hosts_file,
+      family = conf.resolver_family,
+      valid_ttl = conf.resolver_valid_ttl,
+      error_ttl = conf.resolver_error_ttl,
+      stale_ttl = conf.resolver_stale_ttl,
+      cache_size = conf.resolver_lru_cache_size,
+      enable_ipv6 = true, -- allow for IPv6 nameserver addresses
+    }
+  end
+
   assert(dns_client.init(opts))
 
   return dns_client

spec/01-unit/03-conf_loader_spec.lua

@@ -642,8 +642,10 @@ describe("Configuration loader", function()
       local conf = assert(conf_loader())
       assert.same({"bundled"}, conf.plugins)
       assert.same({"LAST", "SRV", "A", "CNAME"}, conf.dns_order)
+      assert.same({"A", "SRV"}, conf.resolver_family)
       assert.is_nil(getmetatable(conf.plugins))
       assert.is_nil(getmetatable(conf.dns_order))
+      assert.is_nil(getmetatable(conf.resolver_family))
     end)
     it("trims array values", function()
       local conf = assert(conf_loader("spec/fixtures/to-strip.conf"))
@@ -788,6 +790,31 @@ describe("Configuration loader", function()
       assert.is_nil(err)
       assert.is_table(conf)
     end)
+    it("errors when resolver_address is not a list in ipv4/6[:port] format (new dns)", function()
+      local conf, err = conf_loader(nil, {
+        resolver_address = "1.2.3.4:53;4.3.2.1" -- ; as separator
+      })
+      assert.equal("resolver_address must be a comma separated list in the form of IPv4/6 or IPv4/6:port, got '1.2.3.4:53;4.3.2.1'", err)
+      assert.is_nil(conf)
+
+      conf, err = conf_loader(nil, {
+        resolver_address = "198.51.100.0:53"
+      })
+      assert.is_nil(err)
+      assert.is_table(conf)
+
+      conf, err = conf_loader(nil, {
+        resolver_address = "[::1]:53"
+      })
+      assert.is_nil(err)
+      assert.is_table(conf)
+
+      conf, err = conf_loader(nil, {
+        resolver_address = "198.51.100.0,1.2.3.4:53,::1,[::1]:53"
+      })
+      assert.is_nil(err)
+      assert.is_table(conf)
+    end)
     it("errors when node_id is not a valid uuid", function()
       local conf, err = conf_loader(nil, {
         node_id = "foobar",
@@ -810,6 +837,15 @@ describe("Configuration loader", function()
       assert.equal([[dns_hostsfile: file does not exist]], err)
       assert.is_nil(conf)
     end)
+    it("errors when the hosts file does not exist (new dns)", function()
+      -- new dns
+      local tmpfile = "/a_file_that_does_not_exist"
+      local conf, err = conf_loader(nil, {
+        resolver_hosts_file = tmpfile,
+      })
+      assert.equal([[resolver_hosts_file: file does not exist]], err)
+      assert.is_nil(conf)
+    end)
     it("accepts an existing hosts file", function()
       local tmpfile = require("pl.path").tmpname()  -- this creates the file!
       finally(function() os.remove(tmpfile) end)
@@ -819,13 +855,29 @@ describe("Configuration loader", function()
       assert.is_nil(err)
       assert.equal(tmpfile, conf.dns_hostsfile)
     end)
+    it("accepts an existing hosts file (new dns)", function()
+      local tmpfile = require("pl.path").tmpname()  -- this creates the file!
+      finally(function() os.remove(tmpfile) end)
+      local conf, err = conf_loader(nil, {
+        resolver_hosts_file = tmpfile,
+      })
+      assert.is_nil(err)
+      assert.equal(tmpfile, conf.resolver_hosts_file)
+    end)
     it("errors on bad entries in the order list", function()
       local conf, err = conf_loader(nil, {
         dns_order = "A,CXAME,SRV",
       })
       assert.is_nil(conf)
       assert.equal([[dns_order: invalid entry 'CXAME']], err)
     end)
+    it("errors on bad entries in the family list", function()
+      local conf, err = conf_loader(nil, {
+        resolver_family = "A,AAAX,SRV",
+      })
+      assert.is_nil(conf)
+      assert.equal([[resolver_family: invalid entry 'AAAX']], err)
+    end)
     it("errors on bad entries in headers", function()
       local conf, err = conf_loader(nil, {
         headers = "server_tokens,Foo-Bar",

spec/02-integration/02-cmd/11-config_spec.lua

@@ -104,6 +104,7 @@ describe("kong config", function()
     assert(helpers.start_kong({
       nginx_conf = "spec/fixtures/custom_nginx.template",
       dns_hostsfile = dns_hostsfile,
+      resolver_hosts_file = dns_hostsfile,
       anonymous_reports = "on",
     }))
 

spec/02-integration/04-admin_api/11-reports_spec.lua

@@ -61,6 +61,7 @@ for _, strategy in helpers.each_strategy() do
         nginx_conf = "spec/fixtures/custom_nginx.template",
         database = strategy,
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
         anonymous_reports = "on",
         declarative_config = yaml_file,
       }, {"routes", "services"}))

spec/02-integration/05-proxy/22-reports_spec.lua

@@ -177,6 +177,7 @@ for _, strategy in helpers.each_strategy() do
         nginx_conf = "spec/fixtures/custom_nginx.template",
         database = strategy,
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
         anonymous_reports = true,
         plugins = "reports-api",
         stream_listen = helpers.get_proxy_ip(false) .. ":19000," ..

spec/02-integration/10-go_plugins/01-reports_spec.lua

@@ -50,6 +50,7 @@ for _, strategy in helpers.each_strategy() do
         nginx_conf = "spec/fixtures/custom_nginx.template",
         database = strategy,
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
         plugins = "bundled,reports-api,go-hello",
         pluginserver_names = "test",
         pluginserver_test_socket = kong_prefix .. "/go-hello.socket",

spec/02-integration/17-admin_gui/03-reports_spec.lua

@@ -69,6 +69,7 @@ describe("anonymous reports for kong manager", function ()
         anonymous_reports = true,
         plugins = "bundled,reports-api",
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
       }))
 
       finally(function()
@@ -84,6 +85,7 @@ describe("anonymous reports for kong manager", function ()
         anonymous_reports = true,
         plugins = "bundled,reports-api",
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
       }))
 
       finally(function()
@@ -101,6 +103,7 @@ describe("anonymous reports for kong manager", function ()
         anonymous_reports = true,
         plugins = "bundled,reports-api",
         dns_hostsfile = dns_hostsfile,
+        resolver_hosts_file = dns_hostsfile,
       }))
 
       local gui_dir_path = prepare_gui_dir()

spec/02-integration/20-wasm/04-proxy-wasm_spec.lua

@@ -138,6 +138,7 @@ describe("proxy-wasm filters (#wasm) (#" .. strategy .. ")", function()
       nginx_conf = "spec/fixtures/custom_nginx.template",
       wasm = true,
       dns_hostsfile = hosts_file,
+      resolver_hosts_file = hosts_file,
       plugins = "pre-function,post-function",
     }))
   end)