Skip to content

Commit

Permalink
tproxy: E2E tests (#20296)
Browse files Browse the repository at this point in the history
Add the `consul-cni` plugin to the Linux AMI for E2E, and add a test case that
covers the transparent proxy feature. Add test assertions to the Connect tests
for upstream reachability

Ref: #20175
  • Loading branch information
tgross authored and philrenaud committed Apr 18, 2024
1 parent 1eac062 commit 987d844
Show file tree
Hide file tree
Showing 4 changed files with 182 additions and 49 deletions.
58 changes: 24 additions & 34 deletions e2e/connect/acls_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,40 +36,39 @@ func TestConnect_LegacyACLs(t *testing.T) {
t.Run("ConnectTerminatingGateway", testConnectTerminatingGatewayLegacyACLs)
}

func createPolicy(t *testing.T, cc *capi.Client, ns, rules string) (string, func()) {
func createPolicy(t *testing.T, cc *capi.Client, ns, rules string) string {
policy, _, err := cc.ACL().PolicyCreate(&capi.ACLPolicy{
Name: "nomad-operator-policy-" + uuid.Short(),
Rules: rules,
Namespace: ns,
}, nil)
must.NoError(t, err)
return policy.ID, func() { cc.ACL().PolicyDelete(policy.ID, nil) }
t.Cleanup(func() { cc.ACL().PolicyDelete(policy.ID, nil) })
return policy.ID
}

func createToken(t *testing.T, cc *capi.Client, policyID, ns string) (string, func()) {
func createToken(t *testing.T, cc *capi.Client, policyID, ns string) string {
token, _, err := cc.ACL().TokenCreate(&capi.ACLToken{
Description: "test token",
Policies: []*capi.ACLTokenPolicyLink{{ID: policyID}},
Namespace: ns,
}, nil)
must.NoError(t, err)
return token.SecretID, func() { cc.ACL().TokenDelete(token.AccessorID, nil) }
t.Cleanup(func() { cc.ACL().TokenDelete(token.AccessorID, nil) })
return token.SecretID
}

// testConnectDemoLegacyACLs tests the demo job file used in Connect Integration examples.
func testConnectDemoLegacyACLs(t *testing.T) {
cc := e2eutil.ConsulClient(t)

policyID, policyCleanup := createPolicy(t, cc, "default",
policyID := createPolicy(t, cc, "default",
`service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`)
t.Cleanup(policyCleanup)

token, tokenCleanup := createToken(t, cc, policyID, "default")
t.Cleanup(tokenCleanup)
token := createToken(t, cc, policyID, "default")

_, cleanup := jobs3.Submit(t, "./input/demo.nomad",
sub, _ := jobs3.Submit(t, "./input/demo.nomad",
jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
t.Cleanup(cleanup)

ixn := &capi.Intention{
SourceName: "count-dashboard",
Expand All @@ -89,6 +88,9 @@ func testConnectDemoLegacyACLs(t *testing.T) {
assertSITokens(t, cc, map[string]int{
"connect-proxy-count-api": 1, "connect-proxy-count-dashboard": 1})

logs := sub.Exec("dashboard", "dashboard",
[]string{"/bin/sh", "-c", "wget -O /dev/null http://${NOMAD_UPSTREAM_ADDR_count_api}"})
must.StrContains(t, logs.Stderr, "saving to")
}

// testConnectDemoLegacyACLsNamespaced tests the demo job file used in Connect
Expand All @@ -101,16 +103,13 @@ func testConnectDemoLegacyACLsNamespaced(t *testing.T) {
must.NoError(t, err)
t.Cleanup(func() { cc.Namespaces().Delete(ns, nil) })

policyID, policyCleanup := createPolicy(t, cc, ns,
policyID := createPolicy(t, cc, ns,
`service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`)
t.Cleanup(policyCleanup)

token, tokenCleanup := createToken(t, cc, policyID, ns)
t.Cleanup(tokenCleanup)
token := createToken(t, cc, policyID, ns)

_, cleanup := jobs3.Submit(t, "./input/demo.nomad",
jobs3.Submit(t, "./input/demo.nomad",
jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
t.Cleanup(cleanup)

ixn := &capi.Intention{
SourceName: "count-dashboard",
Expand All @@ -137,16 +136,13 @@ func testConnectDemoLegacyACLsNamespaced(t *testing.T) {
func testConnectNativeDemoLegacyACLs(t *testing.T) {
cc := e2eutil.ConsulClient(t)

policyID, policyCleanup := createPolicy(t, cc, "default",
policyID := createPolicy(t, cc, "default",
`service "uuid-fe" { policy = "write" } service "uuid-api" { policy = "write" }`)
t.Cleanup(policyCleanup)

token, tokenCleanup := createToken(t, cc, policyID, "default")
t.Cleanup(tokenCleanup)
token := createToken(t, cc, policyID, "default")

_, cleanup := jobs3.Submit(t, "./input/native-demo.nomad",
jobs3.Submit(t, "./input/native-demo.nomad",
jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
t.Cleanup(cleanup)

assertSITokens(t, cc, map[string]int{"frontend": 1, "generate": 1})
}
Expand All @@ -155,16 +151,13 @@ func testConnectNativeDemoLegacyACLs(t *testing.T) {
func testConnectIngressGatewayDemoLegacyACLs(t *testing.T) {
cc := e2eutil.ConsulClient(t)

policyID, policyCleanup := createPolicy(t, cc, "default",
policyID := createPolicy(t, cc, "default",
`service "my-ingress-service" { policy = "write" } service "uuid-api" { policy = "write" }`)
t.Cleanup(policyCleanup)

token, tokenCleanup := createToken(t, cc, policyID, "default")
t.Cleanup(tokenCleanup)
token := createToken(t, cc, policyID, "default")

_, cleanup := jobs3.Submit(t, "./input/ingress-gateway.nomad",
jobs3.Submit(t, "./input/ingress-gateway.nomad",
jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
t.Cleanup(cleanup)

assertSITokens(t, cc, map[string]int{"connect-ingress-my-ingress-service": 1, "generate": 1})
}
Expand All @@ -173,16 +166,13 @@ func testConnectIngressGatewayDemoLegacyACLs(t *testing.T) {
func testConnectTerminatingGatewayLegacyACLs(t *testing.T) {
cc := e2eutil.ConsulClient(t)

policyID, policyCleanup := createPolicy(t, cc, "default",
policyID := createPolicy(t, cc, "default",
`service "api-gateway" { policy = "write" } service "count-dashboard" { policy = "write" }`)
t.Cleanup(policyCleanup)

token, tokenCleanup := createToken(t, cc, policyID, "default")
t.Cleanup(tokenCleanup)
token := createToken(t, cc, policyID, "default")

_, cleanup := jobs3.Submit(t, "./input/terminating-gateway.nomad",
jobs3.Submit(t, "./input/terminating-gateway.nomad",
jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
t.Cleanup(cleanup)

ixn := &capi.Intention{
SourceName: "count-dashboard",
Expand Down
53 changes: 39 additions & 14 deletions e2e/connect/connect_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ func TestConnect(t *testing.T) {
t.Run("ConnectMultiIngress", testConnectMultiIngressGateway)
t.Run("ConnectTerminatingGateway", testConnectTerminatingGateway)
t.Run("ConnectMultiService", testConnectMultiService)
t.Run("ConnectTransparentProxy", testConnectTransparentProxy)
}

// testConnectDemo tests the demo job file used in Connect Integration examples.
func testConnectDemo(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/demo.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
sub, _ := jobs3.Submit(t, "./input/demo.nomad", jobs3.Timeout(time.Second*60))

cc := e2eutil.ConsulClient(t)

Expand All @@ -56,38 +56,37 @@ func testConnectDemo(t *testing.T) {

assertServiceOk(t, cc, "count-api-sidecar-proxy")
assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")

logs := sub.Exec("dashboard", "dashboard",
[]string{"/bin/sh", "-c", "wget -O /dev/null http://${NOMAD_UPSTREAM_ADDR_count_api}"})
must.StrContains(t, logs.Stderr, "saving to")
}

// testConnectCustomSidecarExposed tests that a connect sidecar with custom task
// definition can also make use of the expose service check feature.
func testConnectCustomSidecarExposed(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/expose-custom.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/expose-custom.nomad", jobs3.Timeout(time.Second*60))
}

// testConnectNativeDemo tests the demo job file used in Connect Native
// Integration examples.
func testConnectNativeDemo(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/native-demo.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/native-demo.nomad", jobs3.Timeout(time.Second*60))
}

// testConnectIngressGatewayDemo tests a job with an ingress gateway
func testConnectIngressGatewayDemo(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/ingress-gateway.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/ingress-gateway.nomad", jobs3.Timeout(time.Second*60))
}

// testConnectMultiIngressGateway tests a job with multiple ingress gateways
func testConnectMultiIngressGateway(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/multi-ingress.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/multi-ingress.nomad", jobs3.Timeout(time.Second*60))
}

// testConnectTerminatingGateway tests a job with a terminating gateway
func testConnectTerminatingGateway(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/terminating-gateway.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/terminating-gateway.nomad", jobs3.Timeout(time.Second*60))

cc := e2eutil.ConsulClient(t)

Expand All @@ -112,14 +111,40 @@ func testConnectTerminatingGateway(t *testing.T) {
// testConnectMultiService tests a job with multiple Connect blocks in the same
// group
func testConnectMultiService(t *testing.T) {
_, cleanup := jobs3.Submit(t, "./input/multi-service.nomad", jobs3.Timeout(time.Second*60))
t.Cleanup(cleanup)
jobs3.Submit(t, "./input/multi-service.nomad", jobs3.Timeout(time.Second*60))

cc := e2eutil.ConsulClient(t)
assertServiceOk(t, cc, "echo1-sidecar-proxy")
assertServiceOk(t, cc, "echo2-sidecar-proxy")
}

// testConnectTransparentProxy tests the Connect Transparent Proxy integration
func testConnectTransparentProxy(t *testing.T) {
sub, _ := jobs3.Submit(t, "./input/tproxy.nomad.hcl", jobs3.Timeout(time.Second*60))

cc := e2eutil.ConsulClient(t)

ixn := &capi.Intention{
SourceName: "count-dashboard",
DestinationName: "count-api",
Action: "allow",
}
_, err := cc.Connect().IntentionUpsert(ixn, nil)
must.NoError(t, err, must.Sprint("could not create intention"))

t.Cleanup(func() {
_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
test.NoError(t, err)
})

assertServiceOk(t, cc, "count-api-sidecar-proxy")
assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")

logs := sub.Exec("dashboard", "dashboard",
[]string{"wget", "-O", "/dev/null", "count-api.virtual.consul"})
must.StrContains(t, logs.Stderr, "saving to")
}

// assertServiceOk is a test helper to assert a service is passing health checks, if any
func assertServiceOk(t *testing.T, cc *capi.Client, name string) {
t.Helper()
Expand Down
99 changes: 99 additions & 0 deletions e2e/connect/input/tproxy.nomad.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1

job "countdash" {

constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}

group "api" {
network {
mode = "bridge"
}

service {
name = "count-api"
port = "9001"

check {
type = "http"
path = "/health"
expose = true
interval = "3s"
timeout = "1s"

check_restart {
limit = 0
}
}

connect {
sidecar_service {
proxy {
transparent_proxy {}
}
}
}
}

task "web" {
driver = "docker"

config {
image = "hashicorpdev/counter-api:v3"
auth_soft_fail = true
}
}
}

group "dashboard" {
network {
mode = "bridge"

port "http" {
static = 9010
to = 9002
}
}

service {
name = "count-dashboard"
port = "9002"

check {
type = "http"
path = "/health"
expose = true
interval = "3s"
timeout = "1s"

check_restart {
limit = 0
}
}

connect {
sidecar_service {
proxy {
transparent_proxy {}
}
}
}
}

task "dashboard" {
driver = "docker"

env {
COUNTING_SERVICE_URL = "http://count-api.virtual.consul"
}

config {
image = "hashicorpdev/counter-dashboard:v3"
auth_soft_fail = true
}
}
}
}
21 changes: 20 additions & 1 deletion e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selecti

mkdir_for_root /opt
mkdir_for_root /srv/data # for host volumes
mkdir_for_root /opt/cni/bin

# Dependencies
sudo apt-get update
Expand Down Expand Up @@ -63,6 +64,25 @@ sudo apt-get install -y \
consul-enterprise \
nomad

# TODO(tgross: replace with downloading the binary from releases.hashicorp.com
# once the official 1.4.2 release has shipped
echo "Installing consul-cni plugin"
sudo apt-get install -y build-essential git

pushd /tmp
curl -LO https://go.dev/dl/go1.22.2.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz
git clone --depth=1 https://github.com/hashicorp/consul-k8s.git
pushd consul-k8s
export PATH="$PATH:/usr/local/go/bin"
make control-plane-dev

sudo mv control-plane/cni/bin/consul-cni /opt/cni/bin
sudo chown root:root /opt/cni/bin/consul-cni
sudo chmod +x /opt/cni/bin/consul-cni
popd
popd

# Note: neither service will start on boot because we haven't enabled
# the systemd unit file and we haven't uploaded any configuration
# files for Consul and Nomad
Expand Down Expand Up @@ -90,7 +110,6 @@ sudo apt-get install -y openjdk-17-jdk-headless

# CNI
echo "Installing CNI plugins"
sudo mkdir -p /opt/cni/bin
wget -q -O - \
https://github.com/containernetworking/plugins/releases/download/v1.0.0/cni-plugins-linux-amd64-v1.0.0.tgz \
| sudo tar -C /opt/cni/bin -xz
Expand Down

0 comments on commit 987d844

Please sign in to comment.