[Backport release/3.4.x] feat(balancer): support upstream host_header and router preserve_host config for stream tls proxy

CHANGELOG.md

@@ -38,6 +38,8 @@
 
 - Enable `expressions` and `traditional_compatible` router flavor in stream subsystem.
   [#11071](https://github.com/Kong/kong/pull/11071)
+- Make upstream `host_header` and router `preserve_host` config work in stream tls proxy.
+  [#11244](https://github.com/Kong/kong/pull/11244)
 
 #### Admin API
 

kong/router/atc.lua

@@ -658,6 +658,7 @@ function _M:select(_, _, _, scheme,
       port = service_port,
     },
     upstream_scheme = service_protocol,
+    upstream_host  = matched_route.preserve_host and sni or nil,
   }
 end
 

kong/router/traditional.lua

@@ -1300,6 +1300,10 @@ local function find_match(ctx)
           end
         end
 
+        if matched_route.preserve_host and upstream_host == nil then
+          upstream_host = ctx.sni
+        end
+
         return {
           route           = matched_route.route,
           service         = matched_route.service,

kong/runloop/balancer/init.lua

@@ -28,6 +28,7 @@ local table_concat = table.concat
 local run_hook = hooks.run_hook
 local var = ngx.var
 local get_updated_now_ms = utils.get_updated_now_ms
+local is_http_module   = ngx.config.subsystem == "http"
 
 local CRIT = ngx.CRIT
 local ERR = ngx.ERR
@@ -472,7 +473,11 @@ local function set_host_header(balancer_data, upstream_scheme, upstream_host, is
 
     var.upstream_host = new_upstream_host
 
-    if is_balancer_phase then
+   -- stream module does not support ngx.balancer.recreate_request
+    -- and we do not need to recreate the request in balancer_by_lua
+    -- some nginx proxy variables will compile when init upstream ssl connection
+    -- https://github.com/nginx/nginx/blob/master/src/stream/ngx_stream_proxy_module.c#L1070
+    if is_balancer_phase and is_http_module then
       return recreate_request()
     end
   end

kong/runloop/handler.lua

@@ -1140,13 +1140,36 @@ return {
                        upstream_url_t.host,
                        upstream_url_t.port,
                        service, route)
-      var.upstream_host = upstream_url_t.host
+      if match_t.upstream_host then
+        var.upstream_host = match_t.upstream_host
+      end
     end,
     after = function(ctx)
+      local upstream_scheme = var.upstream_scheme
+
+      local balancer_data = ctx.balancer_data
+      balancer_data.scheme = upstream_scheme -- COMPAT: pdk
+
+      -- The content of var.upstream_host is only set by the router if
+      -- preserve_host is true
+      --
+      -- We can't rely on var.upstream_host for balancer retries inside
+      -- `set_host_header` because it would never be empty after the first -- balancer try
+      local upstream_host = var.upstream_host
+      if upstream_host ~= nil and upstream_host ~= "" then
+        balancer_data.preserve_host = true
+      end
+
       local ok, err, errcode = balancer_execute(ctx)
       if not ok then
         return kong.response.error(errcode, err)
       end
+
+      local ok, err = balancer.set_host_header(balancer_data, upstream_scheme, upstream_host)
+      if not ok then
+        log(ERR, "failed to set balancer Host header: ", err)
+        return exit(500)
+      end
     end
   },
   rewrite = {

spec/02-integration/05-proxy/31-stream_tls_spec.lua

@@ -0,0 +1,161 @@
+local helpers = require "spec.helpers"
+
+for _, flavor in ipairs({ "traditional", "traditional_compatible" }) do
+for _, strategy in helpers.each_strategy({"postgres"}) do
+  describe("#stream Proxying [#" .. strategy .. "] [#" .. flavor .. "]", function()
+    local bp
+    local admin_client
+
+    before_each(function()
+      bp = helpers.get_db_utils(strategy, {
+        "routes",
+        "services",
+        "upstreams",
+        "plugins",
+      }, {
+        "logger",
+      })
+
+      local upstream_srv = bp.upstreams:insert({
+        name = "upstream_srv",
+      })
+
+      bp.targets:insert {
+        target = helpers.mock_upstream_host .. ":" ..
+                 helpers.mock_upstream_stream_ssl_port,
+        upstream = { id = upstream_srv.id },
+      }
+
+      local tls_srv = bp.services:insert({
+        name = "tls",
+        url = "tls://upstream_srv",
+      })
+
+      bp.routes:insert {
+        name = "routes_stream",
+        destinations = {
+          {
+            port = 19443,
+          },
+        },
+        protocols = {
+          "tls",
+        },
+        service = tls_srv,
+      }
+
+      bp.plugins:insert {
+        name = "logger",
+      }
+
+      assert(helpers.start_kong({
+        database   = strategy,
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        plugins = "logger",
+        stream_listen = helpers.get_proxy_ip(false) .. ":19000,"
+          .. helpers.get_proxy_ip(false) .. ":19001,"
+          .. helpers.get_proxy_ip(false) .. ":19002,"
+          .. helpers.get_proxy_ip(false) .. ":19003,"
+          .. helpers.get_proxy_ip(false) .. ":19443 ssl",
+        proxy_stream_error_log = "/tmp/error.log",
+        router_flavor = flavor,
+      }))
+      admin_client = helpers.http_client("127.0.0.1", 9001)
+    end)
+
+    after_each(function()
+      admin_client:close()
+      helpers.stop_kong()
+    end)
+
+    it("tls not set host_header", function()
+      local tcp = ngx.socket.tcp()
+      assert(tcp:connect(helpers.get_proxy_ip(true), 19443))
+      assert(tcp:sslhandshake(nil, "ssl-hello.com", false))
+      assert(tcp:send("get_sni\n"))
+      local body = assert(tcp:receive("*a"))
+      assert.equal("nil\n", body)
+      tcp:close()
+    end)
+
+    it("tls set preserve_host", function()
+      local res = assert(admin_client:send {
+        method  = "PATCH",
+        path    = "/routes/routes_stream",
+        body    = {
+          preserve_host = true,
+        },
+        headers = {
+          ["Content-Type"] = "application/json"
+        }
+      })
+      assert.res_status(200, res)
+      local opt = {
+        stream_enabled = true,
+        stream_ip = "127.0.0.1",
+        stream_port = 19003,
+        timeout = 60,
+      }
+      helpers.wait_for_all_config_update(opt)
+
+      local tcp = ngx.socket.tcp()
+      assert(tcp:connect(helpers.get_proxy_ip(true), 19443))
+      assert(tcp:sslhandshake(nil, "ssl-hello.com", false))
+      assert(tcp:send("get_sni\n"))
+      local body = assert(tcp:receive("*a"))
+      assert.equal("ssl-hello.com\n", body)
+      tcp:close()
+    end)
+
+    it("tls set host_header", function()
+      -- clear preserve_host
+      local res = assert(admin_client:send {
+        method  = "PATCH",
+        path    = "/routes/routes_stream",
+        body    = {
+          preserve_host = false,
+        },
+        headers = {
+          ["Content-Type"] = "application/json"
+        }
+      })
+      assert.res_status(200, res)
+
+      local opt = {
+        stream_enabled = true,
+        stream_port = 19003
+      }
+      helpers.wait_for_all_config_update(opt)
+
+      local tcp = ngx.socket.tcp()
+      assert(tcp:connect(helpers.get_proxy_ip(true), 19443))
+      assert(tcp:sslhandshake(nil, "ssl-hello.com", false))
+      assert(tcp:send("get_sni\n"))
+      local body = assert(tcp:receive("*a"))
+      assert.equal("nil\n", body)
+      tcp:close()
+
+      local res = assert(admin_client:send {
+        method  = "PATCH",
+        path    = "/upstreams/upstream_srv",
+        body    = {
+          host_header = "ssl-hello.com"
+        },
+        headers = {
+          ["Content-Type"] = "application/json"
+        }
+      })
+      assert.res_status(200, res)
+      helpers.wait_for_all_config_update(opt)
+
+      local tcp = ngx.socket.tcp()
+      assert(tcp:connect(helpers.get_proxy_ip(true), 19443))
+      assert(tcp:sslhandshake(nil, "ssl-hello.com", false))
+      assert(tcp:send("get_sni\n"))
+      local body = assert(tcp:receive("*a"))
+      assert.equal("ssl-hello.com\n", body)
+      tcp:close()
+    end)
+  end)
+end
+end

spec/fixtures/custom_nginx.template

@@ -1134,6 +1134,12 @@ server {
             local sock = assert(ngx.req.socket())
             local data = sock:receive()  -- read a line from downstream
 
+            if string.find(data, "get_sni") then
+                sock:send(ngx.var.ssl_server_name)
+                sock:send("\n")
+                return
+            end
+
             if ngx.var.protocol == "TCP" then
                 ngx.say(data)