Option to send websockets upstream using HTTP/1.1

acceptance-tests/acceptance_tests_suite_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gorilla/websocket"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/gbytes"
@@ -33,6 +34,8 @@ var _ = AfterSuite(func() {
 	deleteDeployment(defaultDeploymentName)
 })
 
+var upgrader = websocket.Upgrader{}
+
 // Starts a simple test server that returns 200 OK
 func startDefaultTestServer() (func(), int) {
 	By("Starting a local http server to act as a backend")
@@ -43,6 +46,30 @@ func startDefaultTestServer() (func(), int) {
 	return closeLocalServer, localPort
 }
 
+// Starts a simple test webocket server that echoes back anything sent
+func startDefaultWebsocketServer() (func(), int) {
+	By("Starting a local websocket server to act as a backend")
+	closeLocalServer, localPort, err := startLocalHTTPServer(func(w http.ResponseWriter, r *http.Request) {
+		c, err := upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			return
+		}
+		defer c.Close()
+		for {
+			mt, message, err := c.ReadMessage()
+			if err != nil {
+				break
+			}
+			err = c.WriteMessage(mt, message)
+			if err != nil {
+				break
+			}
+		}
+	})
+	Expect(err).NotTo(HaveOccurred())
+	return closeLocalServer, localPort
+}
+
 // Sets up SSH tunnel from HAProxy VM to test server
 func setupTunnelFromHaproxyToTestServer(haproxyInfo haproxyInfo, haproxyBackendPort, localPort int) func() {
 	By(fmt.Sprintf("Creating a reverse SSH tunnel from HAProxy backend (port %d) to local HTTP server (port %d)", haproxyBackendPort, localPort))

acceptance-tests/go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/bramvdbogaerde/go-scp v0.0.0-20210527193300-acf430e39785
+	github.com/gorilla/websocket v1.4.2
 	github.com/onsi/ginkgo v1.16.2
 	github.com/onsi/gomega v1.13.0
 	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a

acceptance-tests/go.sum

@@ -19,8 +19,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
@@ -80,7 +81,6 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

acceptance-tests/websocket_test.go

@@ -0,0 +1,156 @@
+package acceptance_tests
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+
+	"github.com/gorilla/websocket"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("HTTPS Frontend", func() {
+	var haproxyInfo haproxyInfo
+	var closeTunnel func()
+	var closeLocalServer func()
+	var enableHTTP2 bool
+	var disableBackendHttp2Websockets bool
+	var http1Client *http.Client
+
+	haproxyBackendPort := 12000
+	opsfileHTTPS := `---
+# Configure HTTP2
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/enable_http2?
+  value: ((enable_http2))
+# Configure Disabling Backend HTTP2 Websockets
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/disable_backend_http2_websockets?
+  value: ((disable_backend_http2_websockets))
+# Configure CA and cert chain
+- type: replace
+  path: /instance_groups/name=haproxy/jobs/name=haproxy/properties/ha_proxy/crt_list?/-
+  value:
+    snifilter:
+    - haproxy.internal
+    ssl_pem:
+      cert_chain: ((https_frontend.certificate))((https_frontend_ca.certificate))
+      private_key: ((https_frontend.private_key))
+# Declare certs
+- type: replace
+  path: /variables?/-
+  value:
+    name: https_frontend_ca
+    type: certificate
+    options:
+      is_ca: true
+      common_name: bosh
+- type: replace
+  path: /variables?/-
+  value:
+    name: https_frontend
+    type: certificate
+    options:
+      ca: https_frontend_ca
+      common_name: haproxy.internal
+      alternative_names: [haproxy.internal]
+`
+
+	var creds struct {
+		HTTPSFrontend struct {
+			Certificate string `yaml:"certificate"`
+			PrivateKey  string `yaml:"private_key"`
+			CA          string `yaml:"ca"`
+		} `yaml:"https_frontend"`
+	}
+
+	JustBeforeEach(func() {
+		var varsStoreReader varsStoreReader
+		haproxyInfo, varsStoreReader = deployHAProxy(baseManifestVars{
+			haproxyBackendPort:    haproxyBackendPort,
+			haproxyBackendServers: []string{"127.0.0.1"},
+			deploymentName:        defaultDeploymentName,
+		}, []string{opsfileHTTPS}, map[string]interface{}{
+			"enable_http2":                     enableHTTP2,
+			"disable_backend_http2_websockets": disableBackendHttp2Websockets,
+		}, true)
+
+		err := varsStoreReader(&creds)
+		Expect(err).NotTo(HaveOccurred())
+
+		var localPort int
+		closeLocalServer, localPort = startDefaultWebsocketServer()
+		closeTunnel = setupTunnelFromHaproxyToTestServer(haproxyInfo, haproxyBackendPort, localPort)
+
+		http1Client = buildHTTPClient(
+			[]string{creds.HTTPSFrontend.CA},
+			map[string]string{"haproxy.internal:443": fmt.Sprintf("%s:443", haproxyInfo.PublicIP)},
+			[]tls.Certificate{}, "",
+		)
+	})
+
+	AfterEach(func() {
+		if closeLocalServer != nil {
+			defer closeLocalServer()
+		}
+		if closeTunnel != nil {
+			defer closeTunnel()
+		}
+	})
+
+	Context("When ha_proxy.disable_backend_http2_websockets is true", func() {
+		BeforeEach(func() {
+			enableHTTP2 = true
+			disableBackendHttp2Websockets = true
+		})
+
+		It("succeeds with a websocket", func() {
+			dialer := websocket.DefaultDialer
+			dialer.NetDialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+				if addr == "haproxy.internal:443" {
+					addr = fmt.Sprintf("%s:443", haproxyInfo.PublicIP)
+				}
+				return (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).DialContext(ctx, network, addr)
+			},
+			ws, _, err := dialer.Dial("wss://haproxy.internal:443", nil)
+			Expect(err).NotTo(HaveOccurred())
+			defer ws.Close()
+
+			for i := 0; i < 10; i++ {
+				err := ws.WriteMessage(websocket.TextMessage, []byte("hello"))
+				Expect(err).NotTo(HaveOccurred())
+				_, p, err := ws.ReadMessage()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(string(p)).To(Equal("hello"))
+			}
+
+			// By("Sending a request to HAProxy using HTTP 1.1")
+			// resp, err := http1Client.Get("https://haproxy.internal:443")
+			// Expect(err).NotTo(HaveOccurred())
+
+			// Expect(resp.ProtoMajor).To(Equal(1))
+
+			// Expect(resp.StatusCode).To(Equal(http.StatusOK))
+			// Eventually(gbytes.BufferReader(resp.Body)).Should(gbytes.Say("Hello cloud foundry"))
+
+			// http2Client := buildHTTP2Client(
+			// 	[]string{creds.HTTPSFrontend.CA},
+			// 	map[string]string{"haproxy.internal:443": fmt.Sprintf("%s:443", haproxyInfo.PublicIP)},
+			// 	[]tls.Certificate{},
+			// )
+
+			// By("Sending a request to HAProxy using HTTP 2")
+			// resp, err = http2Client.Get("https://haproxy.internal:443")
+			// Expect(err).NotTo(HaveOccurred())
+
+			// Expect(resp.ProtoMajor).To(Equal(2))
+
+			// Expect(resp.StatusCode).To(Equal(http.StatusOK))
+			// Eventually(gbytes.BufferReader(resp.Body)).Should(gbytes.Say("Hello cloud foundry"))
+		})
+	})
+})

jobs/haproxy/spec

@@ -244,6 +244,9 @@ properties:
   ha_proxy.disable_tls_13:
     default: false
     description: "Disable TLS 1.3 in HA Proxy"
+  ha_proxy.disable_backend_http2_websockets:
+    default: false
+    description: "Forward websockets to the backend servers using HTTP/1.1, never HTTP/2. Does not apply to custom routed_backend_servers. Works around https://github.com/cloudfoundry/routing-release/issues/230"
 
   ha_proxy.connect_timeout:
     description: "Timeout (in floating point seconds) used on connections from haproxy to a backend, while waiting for the TCP handshake to complete + connection to establish"

jobs/haproxy/templates/haproxy.config.erb

@@ -353,6 +353,14 @@ frontend http-in
     acl ssl_redirect hdr(host),lower,map_end(/var/vcap/jobs/haproxy/config/ssl_redirect.map,false) -m str true
     redirect scheme https code 301 if ssl_redirect
   <%- end -%>
+
+  <%- if p("ha_proxy.disable_backend_http2_websockets") -%>
+    # Send websockets to a backend that forces HTTP/1.1. This avoids bugs in Go & Gorouter's HTTP/2 websocket support
+    # https://github.com/cloudfoundry/routing-release/issues/230
+    acl is_websocket hdr(Upgrade) -i WebSocket
+    acl is_websocket hdr_beg(Host) -i ws
+    use_backend http-routers-ws if is_websocket
+  <%- end -%>
 # }}}
 <% end -%>
 
@@ -481,6 +489,14 @@ frontend https-in
   <%- end -%>
     acl xfp_exists hdr_cnt(X-Forwarded-Proto) gt 0
     http-request add-header X-Forwarded-Proto "https" if ! xfp_exists
+
+  <%- if p("ha_proxy.disable_backend_http2_websockets") -%>
+    # Send websockets to a backend that forces HTTP/1.1. This avoids bugs in Go & Gorouter's HTTP/2 websocket support
+    # https://github.com/cloudfoundry/routing-release/issues/230
+    acl is_websocket hdr(Upgrade) -i WebSocket
+    acl is_websocket hdr_beg(Host) -i ws
+    use_backend http-routers-ws if is_websocket
+  <%- end -%>
 # }}}
 <% end -%>
 
@@ -609,6 +625,14 @@ frontend wss-in
   <%- end -%>
     acl xfp_exists hdr_cnt(X-Forwarded-Proto) gt 0
     http-request add-header X-Forwarded-Proto "https" if ! xfp_exists
+
+  <%- if p("ha_proxy.disable_backend_http2_websockets") -%>
+    # Send websockets to a backend that forces HTTP/1.1. This avoids bugs in Go & Gorouter's HTTP/2 websocket support
+    # https://github.com/cloudfoundry/routing-release/issues/230
+    acl is_websocket hdr(Upgrade) -i WebSocket
+    acl is_websocket hdr_beg(Host) -i ws
+    use_backend http-routers-ws if is_websocket
+  <%- end -%>
 # }}}
 <% end -%>
 
@@ -671,6 +695,66 @@ backend http-routers
   <% end %>
 # }}}
 
+<%- if p("ha_proxy.disable_backend_http2_websockets") -%>
+# Default Websocket Backend {{{
+backend http-routers-ws
+    mode http
+    balance roundrobin
+  <%- if p("ha_proxy.compress_types") != "" -%>
+    compression algo gzip
+    compression type <%= p("ha_proxy.compress_types") %>
+  <%- end -%>
+  <%- if properties.ha_proxy.backend_config -%>
+    <%= p("ha_proxy.backend_config") %>
+  <%- end -%>
+    <%- p('ha_proxy.custom_http_error_files', {}).keys.each do |status_code| -%>
+        errorfile <%= status_code %> /var/vcap/jobs/haproxy/errorfiles/custom<%=status_code%>.http
+    <%- end -%>
+    <%
+  backend_servers = []
+  backend_servers_local = []
+  backend_port = nil
+  if_link("http_backend") do |backend|
+    backend_servers = backend.instances.map(&:address)
+    backend_port = backend.p("port", p("ha_proxy.backend_port"))
+
+    if p("ha_proxy.backend_prefer_local_az")
+      backend_servers_local = backend.instances.select{ |n| n.az == spec.az }.map(&:address)
+    end
+  end.else_if_p("ha_proxy.backend_servers") do |servers|
+    backend_servers = servers
+    backend_port = p("ha_proxy.backend_port")
+  end
+  resolvers = ""
+  if_p("ha_proxy.resolvers") do
+    resolvers = "resolvers default "
+  end
+  backend_crt = ""
+  if_p("ha_proxy.backend_crt") do
+    backend_crt = "crt /var/vcap/jobs/haproxy/config/backend-crt.pem "
+  end
+  backend_ssl = ""
+
+  if p("ha_proxy.backend_ssl").downcase == "verify"
+    backend_ssl = "ssl verify required ca-file /var/vcap/jobs/haproxy/config/backend-ca-certs.pem "
+    if_p("ha_proxy.backend_ssl_verifyhost") do | verify_hostname |
+      backend_ssl += "verifyhost #{verify_hostname} "
+    end
+  elsif p("ha_proxy.backend_ssl").downcase == "noverify"
+    backend_ssl = "ssl verify none "
+  end
+  backend_ssl += "alpn http/1.1"
+-%>
+  <%- if p("ha_proxy.backend_use_http_health") == true  -%>
+    option httpchk GET <%= p("ha_proxy.backend_http_health_uri") %>
+    <%- health_check_options = "port " + p("ha_proxy.backend_http_health_port").to_s -%>
+  <%- end -%>
+  <% backend_servers.each_with_index do |ip, index| %>
+    server node<%= index %> <%= ip %>:<%= backend_port -%> <%= resolvers -%><%= backend_crt -%>check inter 1000 <%= health_check_options %> <%= backend_ssl %><%- if !backend_servers_local.empty? && !backend_servers_local.include?(ip)  -%> backup<%- end -%>
+  <% end %>
+# }}}
+<%- end -%>
+
 # Routed Backends {{{
 <% p('ha_proxy.routed_backend_servers').each do |prefix, data| -%>
   <%- prefix_hash = (Digest::SHA256.hexdigest prefix.to_s)[0..5] -%>