Merge branch 'master' into fix/grpc-gateway-decode

DEVELOPER.md

@@ -438,6 +438,83 @@ how to access (or create) a development container with a well-defined tool and r
 - See [How to create a GitHub codespace](https://docs.github.com/en/codespaces/developing-in-codespaces/creating-a-codespace#creating-a-codespace).
 - See [How to create a VSCode development container](https://code.visualstudio.com/docs/remote/containers#_quick-start-try-a-development-container).
 
+## Debugging Kong Gateway with EmmyLua and IntelliJ IDEA/VSCode
+
+[EmmyLua](https://emmylua.github.io/) is a plugin for IntelliJ IDEA and VSCode that provides Lua language
+support.  It comes with debugger support that makes it possible to set breakpoints in Lua code
+and inspect variables.  Kong Gateway can be debugged using EmmyLua by following these steps:
+
+### Install the IDE
+
+#### IntelliJ IDEA
+
+Download and install IntelliJ IDEA from [here](https://www.jetbrains.com/idea/download/).  Note
+that IntelliJ is a commercial product and requires a paid license after the trial period.
+
+#### VSCode
+
+Download and install MS Visual Studio Code from [here](https://code.visualstudio.com/download).
+
+### Install EmmyLua
+
+#### IntelliJ IDEA
+
+Go to the `Settings`->`Plugins`->`Marketplace` and search for `EmmyLua`.
+Install the plugin.
+
+#### VSCode
+
+Go to the `Settings`->`Extensions` and search for `EmmyLua`.
+Install the plugin (publisher is `Tangzx`).
+
+### Download and install the EmmyLua debugging server
+
+The [EmmyLuaDebugger](https://github.com/EmmyLua/EmmyLuaDebugger) is a standalone C++ program
+that runs on the same machine as Kong Gateway and that mediates between the IDE's
+debugger and the Lua code running in Kong Gateway.  It can be downloaded from
+[GitHub](https://github.com/EmmyLua/EmmyLuaDebugger/releases).  The release
+ZIP file contains a single share library named emmy_core.so (Linux) or emmy_core.dylib (macOS).
+Place this file in a directory that is convenient for you and remember the path.
+
+Depending on your Linux version, you may need to compile
+[EmmyLuaDebugger](https://github.com/EmmyLua/EmmyLuaDebugger) on your
+own system as the release binaries published on GitHub assume a pretty
+recent version of GLIBC to be present.
+
+### Start Kong Gateway with the EmmyLua debugger
+
+To enable the EmmyLua debugger, the `KONG_EMMY_DEBUGGER` environment variable must be set to
+the absolute path of the debugger shared library file when Kong Gateway is started.  It is
+also advisable to start Kong Gateway with only one worker process, as debugging multiple worker
+processes is not supported.  For example:
+
+```shell
+KONG_EMMY_DEBUGGER=/path/to/emmy_core.so KONG_NGINX_WORKER_PROCESSES=1 kong start
+```
+
+### Create debugger configuration
+
+#### IntelliJ IDEA
+
+Go to `Run`->`Edit Configurations` and click the `+` button to add a new
+configuration.  Select `Emmy Debugger(NEW)` as the configuration type.  Enter a descriptive
+name for the configuration, e.g. "Kong Gateway Debug".  Click `OK` to save the configuration.
+
+#### VSCode
+
+Go to `Run`->`Add Configuration` and choose `EmmyLua New Debugger`. Enter a descriptive name
+for the configuration, e.g. "Kong Gateway Debug". Save `launch.json`.
+
+### Start the EmmyLua debugger
+
+To connect the EmmyLua debugger to Kong Gateway, click the `Run`->`Debug` menu item in IntelliJ
+(`Run`->`Start Debugging` in VSCode) and select the configuration that you've just created.  You
+will notice that the restart and stop buttons on the top of your IDE will change to solid green
+and red colors.  You can now set breakpoints in your Lua code and start debugging.  Try setting
+a breakpoint in the global `access` function that is defined `runloop/handler.lua` and send
+a proxy request to the Gateway.  The debugger should stop at the breakpoint and you can
+inspect the variables in the request context.
+
 ## What's next
 
 - Refer to the [Kong Gateway Docs](https://docs.konghq.com/gateway/) for more information.

Makefile

@@ -39,7 +39,7 @@ endif
 .PHONY: install dev \
 	lint test test-integration test-plugins test-all \
 	pdk-phase-check functional-tests \
-	fix-windows release wasm-test-filters
+	fix-windows release wasm-test-filters test-logs
 
 ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 KONG_SOURCE_LOCATION ?= $(ROOT_DIR)
@@ -159,6 +159,9 @@ ifndef test_spec
 endif
 	@$(VENV) $(TEST_CMD) $(test_spec)
 
+test-logs:
+	tail -F servroot/logs/error.log
+
 pdk-phase-checks: dev
 	rm -f t/phase_checks.stats
 	rm -f t/phase_checks.report
@@ -198,3 +201,4 @@ install-legacy:
 	@luarocks make OPENSSL_DIR=$(OPENSSL_DIR) CRYPTO_DIR=$(OPENSSL_DIR) YAML_DIR=$(YAML_DIR)
 
 dev-legacy: remove install-legacy dependencies
+

changelog/unreleased/kong/feat-emmy-debugger.yml

@@ -0,0 +1,4 @@
+message: |
+  Added support for debugging with EmmyLuaDebugger.  This feature is a
+  tech preview and not officially supported by Kong Inc. for now.
+type: feature

changelog/unreleased/kong/feat-hybrid-sync-mixed-route-policy.yml

@@ -0,0 +1,7 @@
+message: |
+  When CP runs with `expressions` flavor:
+  - if mixed config is detected and a lower DP is attached to the CP, no config will be sent at all
+  - if the expression is invalid on CP, no config will be sent at all
+  - if the expression is invalid on lower DP, it will be sent to the DP and DP validation will catch this and communicate back to the CP (this could result in partial config application)
+type: feature
+scope: Core

kong-3.7.0-0.rockspec

@@ -193,6 +193,7 @@ build = {
     ["kong.tools.ip"] = "kong/tools/ip.lua",
     ["kong.tools.http"] = "kong/tools/http.lua",
     ["kong.tools.cjson"] = "kong/tools/cjson.lua",
+    ["kong.tools.emmy_debugger"] = "kong/tools/emmy_debugger.lua",
     ["kong.tools.redis.schema"] = "kong/tools/redis/schema.lua",
 
     ["kong.runloop.handler"] = "kong/runloop/handler.lua",

kong.conf.default

@@ -1659,7 +1659,7 @@
                                  #   is not visible.
                                  # - `expressions`: the DSL based expression router engine
                                  #   will be used under the hood. Traditional router
-                                 #   config interface is not visible and you must write
+                                 #   config interface is still visible, and you could also write
                                  #   Router Expression manually and provide them in the
                                  #   `expression` field on the `Route` object.
                                  # - `traditional`: the pre-3.0 Router engine will be

kong/init.lua

@@ -89,6 +89,7 @@ local process = require "ngx.process"
 local tablepool = require "tablepool"
 local table_new = require "table.new"
 local utils = require "kong.tools.utils"
+local emmy_debugger = require "kong.tools.emmy_debugger"
 local get_ctx_table = require("resty.core.ctx").get_ctx_table
 local admin_gui = require "kong.admin_gui"
 local wasm = require "kong.runloop.wasm"
@@ -793,6 +794,9 @@ end
 
 
 function Kong.init_worker()
+
+  emmy_debugger.init()
+
   local ctx = ngx.ctx
 
   ctx.KONG_PHASE = PHASES.init_worker

kong/tools/emmy_debugger.lua

@@ -0,0 +1,83 @@
+local pl_path = require "pl.path"
+local utils = require "kong.tools.utils"
+
+local debugger = os.getenv("KONG_EMMY_DEBUGGER")
+local emmy_debugger_host = os.getenv("KONG_EMMY_DEBUGGER_HOST") or "localhost"
+local emmy_debugger_port = os.getenv("KONG_EMMY_DEBUGGER_PORT") or 9966
+local emmy_debugger_wait = os.getenv("KONG_EMMY_DEBUGGER_WAIT")
+local emmy_debugger_source_path = utils.split(os.getenv("KONG_EMMY_DEBUGGER_SOURCE_PATH") or "", ":")
+
+local function find_source(path)
+  if pl_path.exists(path) then
+    return path
+  end
+
+  if path:match("^=") then
+    -- code is executing from .conf file, don't attempt to map
+    return path
+  end
+
+  for _, source_path in ipairs(emmy_debugger_source_path) do
+    local full_path = pl_path.join(source_path, path)
+    if pl_path.exists(full_path) then
+      return full_path
+    end
+  end
+
+  ngx.log(ngx.ERR, "source file " .. path .. " not found in KONG_EMMY_DEBUGGER_SOURCE_PATH")
+
+  return path
+end
+
+local function init()
+  if not debugger then
+    return
+  end
+
+  if not pl_path.isabs(debugger) then
+    ngx.log(ngx.ERR, "KONG_EMMY_DEBUGGER (" .. debugger .. ") must be an absolute path")
+    return
+  end
+  if not pl_path.exists(debugger) then
+    ngx.log(ngx.ERR, "KONG_EMMY_DEBUGGER (" .. debugger .. ") file not found")
+    return
+  end
+  local ext = pl_path.extension(debugger)
+  if ext ~= ".so" and ext ~= ".dylib" then
+    ngx.log(ngx.ERR, "KONG_EMMY_DEBUGGER (" .. debugger .. ") must be a .so (Linux) or .dylib (macOS) file")
+    return
+  end
+  if ngx.worker.id() ~= 0 then
+    ngx.log(ngx.ERR, "KONG_EMMY_DEBUGGER is only supported in the first worker process, suggest setting KONG_NGINX_WORKER_PROCESSES to 1")
+    return
+  end
+
+  ngx.log(ngx.NOTICE, "loading EmmyLua debugger " .. debugger)
+  ngx.log(ngx.WARN, "The EmmyLua integration for Kong is a feature solely for your convenience during development. Kong assumes no liability as a result of using the integration and does not endorse it’s usage. Issues related to usage of EmmyLua integration should be directed to the respective project instead.")
+
+  _G.emmy = {
+    fixPath = find_source
+  }
+
+  local name = pl_path.basename(debugger):sub(1, -#ext - 1)
+
+  local save_cpath = package.cpath
+  package.cpath = pl_path.dirname(debugger) .. '/?' .. ext
+  local dbg = require(name)
+  package.cpath = save_cpath
+
+  dbg.tcpListen(emmy_debugger_host, emmy_debugger_port)
+
+  ngx.log(ngx.NOTICE, "EmmyLua debugger loaded, listening on port ", emmy_debugger_port)
+
+  if emmy_debugger_wait then
+    -- Wait for IDE connection
+    ngx.log(ngx.NOTICE, "waiting for IDE to connect")
+    dbg.waitIDE()
+    ngx.log(ngx.NOTICE, "IDE connected")
+  end
+end
+
+return {
+  init = init
+}