feat(wasm): add support for wasmtime cache (#12930)

* feat(wasm): add support for wasmtime cache

This adds support for Wasmtime's module caching.
See also:

* Kong/ngx_wasm_module#540
* https://github.com/Kong/ngx_wasm_module/blob/b19d405403ca6765c548e571010aea3af1accaea/docs/DIRECTIVES.md?plain=1#L136-L149
* https://docs.wasmtime.dev/cli-cache.html

* tests(wasm): add start/restart test for wasmtime cache

changelog/unreleased/kong/wasm-module-cache.yml

@@ -0,0 +1,3 @@
+message: Configure Wasmtime module cache when Wasm is enabled
+type: feature
+scope: Configuration

kong-3.8.0-0.rockspec

@@ -109,6 +109,7 @@ build = {
     ["kong.templates.nginx_kong_inject"] = "kong/templates/nginx_kong_inject.lua",
     ["kong.templates.nginx_kong_stream_inject"] = "kong/templates/nginx_kong_stream_inject.lua",
     ["kong.templates.kong_yml"] = "kong/templates/kong_yml.lua",
+    ["kong.templates.wasmtime_cache_config"] = "kong/templates/wasmtime_cache_config.lua",
 
     ["kong.resty.dns.client"] = "kong/resty/dns/client.lua",
     ["kong.resty.dns.utils"] = "kong/resty/dns/utils.lua",

kong/cmd/utils/prefix_handler.lua

@@ -5,6 +5,7 @@ local kong_nginx_stream_template = require "kong.templates.nginx_kong_stream"
 local nginx_main_inject_template = require "kong.templates.nginx_inject"
 local nginx_http_inject_template = require "kong.templates.nginx_kong_inject"
 local nginx_stream_inject_template = require "kong.templates.nginx_kong_stream_inject"
+local wasmtime_cache_template = require "kong.templates.wasmtime_cache_config"
 local system_constants = require "lua_system_constants"
 local process_secrets = require "kong.cmd.utils.process_secrets"
 local openssl_bignum = require "resty.openssl.bn"
@@ -41,6 +42,7 @@ local math = math
 local join = pl_path.join
 local io = io
 local os = os
+local fmt = string.format
 
 
 local function pre_create_private_file(file)
@@ -235,6 +237,10 @@ local function get_ulimit()
   end
 end
 
+local function quote(s)
+  return fmt("%q", s)
+end
+
 local function compile_conf(kong_config, conf_template, template_env_inject)
   -- computed config properties for templating
   local compile_env = {
@@ -244,7 +250,8 @@ local function compile_conf(kong_config, conf_template, template_env_inject)
     tostring = tostring,
     os = {
       getenv = os.getenv,
-    }
+    },
+    quote = quote,
   }
 
   local kong_proxy_access_log = kong_config.proxy_access_log
@@ -419,6 +426,10 @@ local function compile_nginx_conf(kong_config, template)
   return compile_conf(kong_config, template)
 end
 
+local function compile_wasmtime_cache_conf(kong_config)
+  return compile_conf(kong_config, wasmtime_cache_template)
+end
+
 local function prepare_prefixed_interface_dir(usr_path, interface_dir, kong_config)
   local usr_interface_path = usr_path .. "/" .. interface_dir
   local interface_path = kong_config.prefix .. "/" .. interface_dir
@@ -673,6 +684,23 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     return true
   end
 
+  if kong_config.wasm then
+    if kong_config.wasmtime_cache_directory then
+      local ok, err = makepath(kong_config.wasmtime_cache_directory)
+      if not ok then
+        return nil, err
+      end
+    end
+
+    if kong_config.wasmtime_cache_config_file  then
+      local wasmtime_conf, err = compile_wasmtime_cache_conf(kong_config)
+      if not wasmtime_conf then
+        return nil, err
+      end
+      pl_file.write(kong_config.wasmtime_cache_config_file, wasmtime_conf)
+    end
+  end
+
   -- compile Nginx configurations
   local nginx_template
   if nginx_custom_template_path then

kong/conf_loader/init.lua

@@ -647,6 +647,12 @@ local function load(path, custom_conf, opts)
     -- TODO: as a temporary compatibility fix, we are forcing it to 'off'.
     add_wasm_directive("nginx_http_proxy_wasm_lua_resolver", "off")
 
+    -- configure wasmtime module cache
+    if conf.role == "traditional" or conf.role == "data_plane" then
+      conf.wasmtime_cache_directory = pl_path.join(conf.prefix, ".wasmtime_cache")
+      conf.wasmtime_cache_config_file = pl_path.join(conf.prefix, ".wasmtime_config.toml")
+    end
+
     -- wasm vm properties are inherited from previously set directives
     if conf.lua_ssl_trusted_certificate and #conf.lua_ssl_trusted_certificate >= 1 then
       add_wasm_directive("tls_trusted_certificate", conf.lua_ssl_trusted_certificate[1], wasm_main_prefix)

kong/templates/nginx.lua

@@ -38,8 +38,12 @@ wasm {
 > end
 > end
 
-> if #nginx_wasm_wasmtime_directives > 0 then
+> if #nginx_wasm_wasmtime_directives > 0 or wasmtime_cache_config_file then
   wasmtime {
+> if wasmtime_cache_config_file then
+    cache_config $(quote(wasmtime_cache_config_file));
+> end
+
 > for _, el in ipairs(nginx_wasm_wasmtime_directives) do
     flag $(el.name) $(el.value);
 > end

kong/templates/wasmtime_cache_config.lua

@@ -0,0 +1,10 @@
+return [[
+# *************************
+# * DO NOT EDIT THIS FILE *
+# *************************
+# This configuration file is auto-generated.
+# Any modifications made here will be lost.
+[cache]
+enabled = true
+directory = $(quote(wasmtime_cache_directory))
+]]

spec/01-unit/03-conf_loader_spec.lua

@@ -2105,6 +2105,19 @@ describe("Configuration loader", function()
       assert.is_true(found, "expected the user filter to be enabled")
     end)
 
+    it("populates wasmtime_cache_* properties", function()
+      local conf, err = conf_loader(nil, {
+        wasm = "on",
+        wasm_filters = "bundled,user",
+        wasm_filters_path = temp_dir,
+      })
+      assert.is_nil(err)
+
+      assert.is_string(conf.wasmtime_cache_directory,
+                       "wasmtime_cache_directory was not set")
+      assert.is_string(conf.wasmtime_cache_config_file,
+                       "wasmtime_cache_config_file was not set")
+    end)
   end)
 
   describe("errors", function()

spec/01-unit/04-prefix_handler_spec.lua

@@ -1024,6 +1024,14 @@ describe("NGINX conf compiler", function()
           }, debug)
         )
       end)
+      it("injects wasmtime cache_config", function()
+        assert.matches(
+          "wasm {.+wasmtime {.+cache_config .+%.wasmtime_config%.toml.*;",
+          ngx_cfg({
+            wasm = true,
+          }, debug)
+        )
+      end)
       describe("injects inherited directives", function()
         it("only if one isn't explicitly set", function()
           assert.matches(

spec/02-integration/20-wasm/10-wasmtime_spec.lua

@@ -0,0 +1,169 @@
+local helpers = require "spec.helpers"
+local fmt = string.format
+
+for _, role in ipairs({"traditional", "control_plane", "data_plane"}) do
+
+describe("#wasm wasmtime (role: " .. role .. ")", function()
+  describe("kong prepare", function()
+    local conf
+    local prefix = "./wasm"
+
+    lazy_setup(function()
+      helpers.clean_prefix(prefix)
+      assert(helpers.kong_exec("prepare", {
+        database = role == "data_plane" and "off" or "postgres",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        wasm = true,
+        prefix = prefix,
+        role = role,
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      }))
+
+      conf = assert(helpers.get_running_conf(prefix))
+    end)
+
+    lazy_teardown(function()
+      helpers.clean_prefix(prefix)
+    end)
+
+    if role == "control_plane" then
+      it("does not populate wasmtime config values", function()
+        assert.is_nil(conf.wasmtime_cache_directory,
+                         "wasmtime_cache_directory should not be set")
+        assert.is_nil(conf.wasmtime_cache_config_file,
+                         "wasmtime_cache_config_file should not be set")
+      end)
+
+    else
+      it("populates wasmtime config values", function()
+        assert.is_string(conf.wasmtime_cache_directory,
+                         "wasmtime_cache_directory was not set")
+        assert.is_string(conf.wasmtime_cache_config_file,
+                         "wasmtime_cache_config_file was not set")
+      end)
+
+      it("creates the cache directory", function()
+        assert(helpers.path.isdir(conf.wasmtime_cache_directory),
+               fmt("expected cache directory (%s) to exist",
+                   conf.wasmtime_cache_directory))
+      end)
+
+      it("creates the cache config file", function()
+        assert(helpers.path.isfile(conf.wasmtime_cache_config_file),
+               fmt("expected cache config file (%s) to exist",
+                   conf.wasmtime_cache_config_file))
+
+        local cache_config = assert(helpers.file.read(conf.wasmtime_cache_config_file))
+        assert.matches(conf.wasmtime_cache_directory, cache_config, nil, true,
+                       "expected cache config file to reference the cache directory")
+      end)
+    end
+  end) -- kong prepare
+
+  describe("kong stop/start/restart", function()
+    local conf
+    local prefix = "./wasm"
+    local log = prefix .. "/logs/error.log"
+    local status_port
+    local client
+    local cp_prefix = "./wasm-cp"
+
+    lazy_setup(function()
+      if role == "traditional" then
+        helpers.get_db_utils("postgres")
+      end
+
+      helpers.clean_prefix(prefix)
+      status_port = helpers.get_available_port()
+
+      assert(helpers.kong_exec("prepare", {
+        database = role == "data_plane" and "off" or "postgres",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        wasm = true,
+        prefix = prefix,
+        role = role,
+        --wasm_filters_path = helpers.test_conf.wasm_filters_path,
+        wasm_filters = "tests,response_transformer",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+
+        status_listen = "127.0.0.1:" .. status_port,
+        nginx_main_worker_processes = 2,
+      }))
+
+      conf = assert(helpers.get_running_conf(prefix))
+
+      -- we need to briefly spin up a control plane, or else we will get
+      -- error.log entries when our data plane tries to connect
+      if role == "data_plane" then
+        helpers.get_db_utils("postgres")
+
+        assert(helpers.start_kong({
+          database = "postgres",
+          nginx_conf = "spec/fixtures/custom_nginx.template",
+          wasm = true,
+          prefix = cp_prefix,
+          role = "control_plane",
+          wasm_filters = "tests,response_transformer",
+          cluster_cert = "spec/fixtures/kong_clustering.crt",
+          cluster_cert_key = "spec/fixtures/kong_clustering.key",
+          status_listen = "off",
+          nginx_main_worker_processes = 2,
+        }))
+      end
+    end)
+
+    lazy_teardown(function()
+      if client then
+        client:close()
+      end
+
+      helpers.stop_kong(prefix)
+
+      if role == "data_plane" then
+        helpers.stop_kong(cp_prefix)
+      end
+    end)
+
+    it("does not introduce any errors", function()
+      local function assert_no_errors()
+        assert.logfile(log).has.no.line("[error]", true, 0)
+        assert.logfile(log).has.no.line("[alert]", true, 0)
+        assert.logfile(log).has.no.line("[emerg]", true, 0)
+        assert.logfile(log).has.no.line("[crit]",  true, 0)
+      end
+
+      local function assert_kong_status(context)
+        if not client then
+          client = helpers.proxy_client(1000, status_port)
+          client.reopen = true
+        end
+
+        assert.eventually(function()
+          local res, err = client:send({ path = "/status", method = "GET" })
+          if res and res.status == 200 then
+            return true
+          end
+
+          return nil, err or "non-200 status"
+        end)
+        .is_truthy("failed waiting for kong status " .. context)
+      end
+
+      assert(helpers.start_kong(conf, nil, true))
+      assert_no_errors()
+
+      assert_kong_status("after fresh startup")
+      assert_no_errors()
+
+      assert(helpers.restart_kong(conf))
+      assert_no_errors()
+
+      assert_kong_status("after restart")
+      assert_no_errors()
+    end)
+  end) -- kong stop/start/restart
+
+end) -- wasmtime
+end -- each role

spec/fixtures/custom_nginx.template

@@ -48,8 +48,12 @@ wasm {
 > end
 > end
 
-> if #nginx_wasm_wasmtime_directives > 0 then
+> if #nginx_wasm_wasmtime_directives > 0 or wasmtime_cache_config_file then
   wasmtime {
+> if wasmtime_cache_config_file then
+    cache_config $(quote(wasmtime_cache_config_file));
+> end
+
 > for _, el in ipairs(nginx_wasm_wasmtime_directives) do
     flag $(el.name) $(el.value);
 > end