E2E: refactor Connect tests to use stdlib testing (#20278)

Migrate our E2E tests for Connect off the old framework in preparation for
writing E2E tests for transparent proxy and the updated workload identity
workflow. Mark the tests that cover the legacy Consul token submitted workflow.

Ref: #20175

e2e/connect/acls_test.go

@@ -4,19 +4,241 @@
 package connect
 
 import (
+	"regexp"
 	"testing"
+	"time"
 
-	"github.com/stretchr/testify/require"
+	capi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/shoenig/test"
+	"github.com/shoenig/test/must"
 )
 
+// TestConnect_LegacyACLs tests the workflows where the operator provides their
+// own token, rather than using Nomad's token or Workload Identity
+func TestConnect_LegacyACLs(t *testing.T) {
+
+	nomadClient := e2eutil.NomadClient(t)
+	e2eutil.WaitForLeader(t, nomadClient)
+	e2eutil.WaitForNodesReady(t, nomadClient, 2)
+
+	t.Cleanup(func() {
+		_, err := e2eutil.Command("nomad", "system", "gc")
+		test.NoError(t, err)
+	})
+
+	t.Run("ConnectDemo", testConnectDemoLegacyACLs)
+	t.Run("ConnectDemoNamespaced", testConnectDemoLegacyACLsNamespaced)
+	t.Run("ConnectNativeDemo", testConnectNativeDemoLegacyACLs)
+	t.Run("ConnectIngressGatewayDemo", testConnectIngressGatewayDemoLegacyACLs)
+	t.Run("ConnectTerminatingGateway", testConnectTerminatingGatewayLegacyACLs)
+}
+
+func createPolicy(t *testing.T, cc *capi.Client, ns, rules string) (string, func()) {
+	policy, _, err := cc.ACL().PolicyCreate(&capi.ACLPolicy{
+		Name:      "nomad-operator-policy-" + uuid.Short(),
+		Rules:     rules,
+		Namespace: ns,
+	}, nil)
+	must.NoError(t, err)
+	return policy.ID, func() { cc.ACL().PolicyDelete(policy.ID, nil) }
+}
+
+func createToken(t *testing.T, cc *capi.Client, policyID, ns string) (string, func()) {
+	token, _, err := cc.ACL().TokenCreate(&capi.ACLToken{
+		Description: "test token",
+		Policies:    []*capi.ACLTokenPolicyLink{{ID: policyID}},
+		Namespace:   ns,
+	}, nil)
+	must.NoError(t, err)
+	return token.SecretID, func() { cc.ACL().TokenDelete(token.AccessorID, nil) }
+}
+
+// testConnectDemoLegacyACLs tests the demo job file used in Connect Integration examples.
+func testConnectDemoLegacyACLs(t *testing.T) {
+	cc := e2eutil.ConsulClient(t)
+
+	policyID, policyCleanup := createPolicy(t, cc, "default",
+		`service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`)
+	t.Cleanup(policyCleanup)
+
+	token, tokenCleanup := createToken(t, cc, policyID, "default")
+	t.Cleanup(tokenCleanup)
+
+	_, cleanup := jobs3.Submit(t, "./input/demo.nomad",
+		jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
+	t.Cleanup(cleanup)
+
+	ixn := &capi.Intention{
+		SourceName:      "count-dashboard",
+		DestinationName: "count-api",
+		Action:          "allow",
+	}
+	_, err := cc.Connect().IntentionUpsert(ixn, nil)
+	must.NoError(t, err, must.Sprint("could not create intention"))
+
+	t.Cleanup(func() {
+		_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
+		test.NoError(t, err)
+	})
+
+	assertServiceOk(t, cc, "count-api-sidecar-proxy")
+	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+	assertSITokens(t, cc, map[string]int{
+		"connect-proxy-count-api": 1, "connect-proxy-count-dashboard": 1})
+
+}
+
+// testConnectDemoLegacyACLsNamespaced tests the demo job file used in Connect
+// Integration examples.
+func testConnectDemoLegacyACLsNamespaced(t *testing.T) {
+	cc := e2eutil.ConsulClient(t)
+
+	ns := "ns-" + uuid.Short()
+	_, _, err := cc.Namespaces().Create(&capi.Namespace{Name: ns}, nil)
+	must.NoError(t, err)
+	t.Cleanup(func() { cc.Namespaces().Delete(ns, nil) })
+
+	policyID, policyCleanup := createPolicy(t, cc, ns,
+		`service "count-api" { policy = "write" } service "count-dashboard" { policy = "write" }`)
+	t.Cleanup(policyCleanup)
+
+	token, tokenCleanup := createToken(t, cc, policyID, ns)
+	t.Cleanup(tokenCleanup)
+
+	_, cleanup := jobs3.Submit(t, "./input/demo.nomad",
+		jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
+	t.Cleanup(cleanup)
+
+	ixn := &capi.Intention{
+		SourceName:      "count-dashboard",
+		DestinationName: "count-api",
+		Action:          "allow",
+	}
+	_, err = cc.Connect().IntentionUpsert(ixn, nil)
+	must.NoError(t, err, must.Sprint("could not create intention"))
+
+	t.Cleanup(func() {
+		_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
+		test.NoError(t, err)
+	})
+
+	assertServiceOk(t, cc, "count-api-sidecar-proxy")
+	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+	assertSITokens(t, cc, map[string]int{
+		"connect-proxy-count-api": 1, "connect-proxy-count-dashboard": 1})
+
+}
+
+// testConnectNativeDemoLegacyACLs tests the demo job file used in Connect Native
+// Integration examples.
+func testConnectNativeDemoLegacyACLs(t *testing.T) {
+	cc := e2eutil.ConsulClient(t)
+
+	policyID, policyCleanup := createPolicy(t, cc, "default",
+		`service "uuid-fe" { policy = "write" } service "uuid-api" { policy = "write" }`)
+	t.Cleanup(policyCleanup)
+
+	token, tokenCleanup := createToken(t, cc, policyID, "default")
+	t.Cleanup(tokenCleanup)
+
+	_, cleanup := jobs3.Submit(t, "./input/native-demo.nomad",
+		jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
+	t.Cleanup(cleanup)
+
+	assertSITokens(t, cc, map[string]int{"frontend": 1, "generate": 1})
+}
+
+// testConnectIngressGatewayDemoLegacyACLs tests a job with an ingress gateway
+func testConnectIngressGatewayDemoLegacyACLs(t *testing.T) {
+	cc := e2eutil.ConsulClient(t)
+
+	policyID, policyCleanup := createPolicy(t, cc, "default",
+		`service "my-ingress-service" { policy = "write" } service "uuid-api" { policy = "write" }`)
+	t.Cleanup(policyCleanup)
+
+	token, tokenCleanup := createToken(t, cc, policyID, "default")
+	t.Cleanup(tokenCleanup)
+
+	_, cleanup := jobs3.Submit(t, "./input/ingress-gateway.nomad",
+		jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
+	t.Cleanup(cleanup)
+
+	assertSITokens(t, cc, map[string]int{"connect-ingress-my-ingress-service": 1, "generate": 1})
+}
+
+// testConnectTerminatingGatewayLegacyACLs tests a job with a terminating gateway
+func testConnectTerminatingGatewayLegacyACLs(t *testing.T) {
+	cc := e2eutil.ConsulClient(t)
+
+	policyID, policyCleanup := createPolicy(t, cc, "default",
+		`service "api-gateway" { policy = "write" } service "count-dashboard" { policy = "write" }`)
+	t.Cleanup(policyCleanup)
+
+	token, tokenCleanup := createToken(t, cc, policyID, "default")
+	t.Cleanup(tokenCleanup)
+
+	_, cleanup := jobs3.Submit(t, "./input/terminating-gateway.nomad",
+		jobs3.Timeout(time.Second*60), jobs3.LegacyConsulToken(token))
+	t.Cleanup(cleanup)
+
+	ixn := &capi.Intention{
+		SourceName:      "count-dashboard",
+		DestinationName: "count-api",
+		Action:          "allow",
+	}
+	_, err := cc.Connect().IntentionUpsert(ixn, nil)
+	must.NoError(t, err, must.Sprint("could not create intention"))
+
+	t.Cleanup(func() {
+		_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
+		test.NoError(t, err)
+	})
+
+	assertServiceOk(t, cc, "api-gateway")
+	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+	assertServiceOk(t, cc, "count-api")
+
+	assertSITokens(t, cc, map[string]int{
+		"connect-terminating-api-gateway": 1, "connect-proxy-count-dashboard": 1})
+}
+
+func assertSITokens(t *testing.T, cc *capi.Client, expect map[string]int) {
+	tokens, _, err := cc.ACL().TokenList(nil)
+	must.NoError(t, err)
+
+	// count the number of SI tokens matching each service name
+	foundSITokens := make(map[string]int)
+	for _, token := range tokens {
+		if service := serviceofSIToken(token.Description); service != "" {
+			foundSITokens[service]++
+		}
+	}
+	for expected, count := range expect {
+		test.Eq(t, count, foundSITokens[expected], test.Sprintf("expected tokens for %q", expected))
+	}
+}
+
 func Test_serviceOfSIToken(t *testing.T) {
 	try := func(description, exp string) {
-		tc := new(ConnectACLsE2ETest)
-		result := tc.serviceofSIToken(description)
-		require.Equal(t, exp, result)
+		result := serviceofSIToken(description)
+		must.Eq(t, exp, result)
 	}
 
 	try("", "")
 	try("foobarbaz", "")
 	try("_nomad_si [8b1a5d3f-7e61-4a5a-8a57-7e7ad91e63b6] [8b1a5d3f-7e61-4a5a-8a57-7e7ad91e63b6] [foo-service]", "foo-service")
 }
+
+var (
+	siTokenRe = regexp.MustCompile(`_nomad_si \[[\w-]{36}] \[[\w-]{36}] \[([\S]+)]`)
+)
+
+func serviceofSIToken(description string) string {
+	if m := siTokenRe.FindStringSubmatch(description); len(m) == 2 {
+		return m[1]
+	}
+	return ""
+}

e2e/connect/client_test.go

@@ -0,0 +1,50 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package connect
+
+import (
+	"testing"
+
+	capi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/v3/jobs3"
+	"github.com/shoenig/test"
+	"github.com/shoenig/test/must"
+)
+
+func TestConnect_ClientRestart(t *testing.T) {
+	t.Skip("skipping test that does nomad agent restart")
+
+	nomadClient := e2eutil.NomadClient(t)
+	e2eutil.WaitForLeader(t, nomadClient)
+	e2eutil.WaitForNodesReady(t, nomadClient, 2)
+
+	sub, cleanup := jobs3.Submit(t, "./input/demo.nomad")
+	t.Cleanup(cleanup)
+
+	cc := e2eutil.ConsulClient(t)
+
+	ixn := &capi.Intention{
+		SourceName:      "count-dashboard",
+		DestinationName: "count-api",
+		Action:          "allow",
+	}
+	_, err := cc.Connect().IntentionUpsert(ixn, nil)
+	must.NoError(t, err, must.Sprint("could not create intention"))
+
+	t.Cleanup(func() {
+		_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
+		test.NoError(t, err)
+	})
+
+	assertServiceOk(t, cc, "count-api-sidecar-proxy")
+	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+
+	nodeID := sub.Allocs()[0].NodeID
+	_, err = e2eutil.AgentRestart(nomadClient, nodeID)
+	must.Error(t, err, must.Sprint("node cannot be restarted"))
+
+	assertServiceOk(t, cc, "count-api-sidecar-proxy")
+	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+}