From cbe80513611c7c27ef714312ad9f05da6c46dfe2 Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Thu, 7 Dec 2023 17:15:00 +0100 Subject: [PATCH] [E2E Testing] mage targets to interact with VMs and cloud stack This commit adds a number of mage targets to improve developer experience when using the integration test framework. Those targets aim to easy how to login into VMs and run tests from there as well as how to access the Cloud Stacks. The targets added are: - integration:deployDebugTools - integration:deployEnvFile - integration:generateEnvFile - integration:listStacks - integration:listVMs - integration:printState - integration:sshvm --- docs/test-framework-dev-guide.md | 80 +++++++ magefile.go | 350 +++++++++++++++++++++++++++++++ 2 files changed, 430 insertions(+) diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index 9e38e093103..5eae4114269 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -126,6 +126,60 @@ when running them manually, such as `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USERNAM ### Debugging tests +#### Manually debugging tests on VMs +Many of the integration tests will install the Elastic-Agent and/or +require root to run, which makes it hard to just run them on our work +machines, the best way to circumvent that is to debug the tests +directly on a VM. `mage integration:DeployDebugTools` will show a menu +to select a VM and then install the common debugging tools: Delve, +Mage and Docker. It will also create the `~/elastic-agent` folder +containing the Git repository (required to package from within the VM) +and the last version of the code uploaded to the VM. This allows you +to easily build/package the Elastic-Agent from within the VM as well +as run any tests. + +After deploying the debug tools, `mage integrationDeployEnvFile` will +create a `env.sh` and copy it to a selected VM, sourcing it will allow +you to any test against the Cloud Stack you selected. + +Example of how to run a test from within the VM: +``` +## Run a single test +SNAPSHOT=true TEST_PLATFORMS="linux/amd64" mage integration:single TestLogIngestionFleetManaged +## Let's suppose it has failed + +## Install DebugTools +mage -v integration:DeployDebugTools + +## Generate and deploy env file +mage -v integration:DeployEnvFile + +## SSH into the VM +$(mage integration:SSHVM) + +## From inside the VM, the test needs root +sudo su +source ./env.sh +cd elastic-agent +## Any flags passed to the test binary go after the '--', they also need to +## include the `test.` prefix if they're for `go test` +TEST_DEFINE_PREFIX=gambiarra dlv test ./testing/integration/ --build-flags="-tags integration" -- -test.v -test.run TestLogIngestionFleetManaged +``` + +**A Delve trick** +If you didn't build the Elastic-Agent directly on the machine you're +debugging, it is very likely the location of the source code is +different, hence delve cannot show you the code it is running. To +solve this, once on Delve shell, run: +`` +config substitute-path /go/src/github.com/elastic/elastic-agent /home/ubuntu/elastic-agent` +`` +where: +- `/go/src/github.com/elastic/elastic-agent` is the path annotated in + the binary you are debugging (the one Delve shows). +- `/home/ubuntu/elastic-agent` is where Delve should read the source + code form. + #### Auto diagnostics retrieval When an integration test fails the testing fixture will try its best to automatically collect the diagnostic information of the installed Elastic Agent. In the case that diagnostics is collected the test runner will @@ -164,6 +218,32 @@ with `go test`. E.g.: TEST_DEFINE_PREFIX=gambiarra go test -v -tags integration -run TestProxyURL ./testing/integration/ ``` +## Connecting to VMs and running tests +### Connecting to VMs +All VMs (including Windows) support connections via SSH, the framework +generates and stores the necessary SSH keys to access the VMs, the +easiest way to connect to them is using the SSH command returned by +`mage integration:SSHVM`. It will list the VMs and ask to select +one. + +On a Unix shell you can run `$(mage integration:SSHVM)`, the menu is +printed to stderr and the SSH command to stdout. After selecting the +VM you will have shell connected to it. + +### Credentials for cloud stack/projects +All cloud deployments and projects can be listed with `mage +integration:listStacks`, they can be used to manually connect to +Kibana and Elasticsearch. + +If you need to manually run tests against any deployments, `mage +integration:GenerateEnvFile` will generate a file called `env.sh` that +exports environment variables for Unix compatible shells, you can load +them into your shell by running `source ./env.sh`. + +To easily deploy the credentials to any VM, just run `mage +integration:DeployEnvFile`. A menu will ask for the desired Stack and +VM. + ## Writing tests Write integration and E2E tests by adding them to the `testing/integration` diff --git a/magefile.go b/magefile.go index bd00b199040..ebca0a9045a 100644 --- a/magefile.go +++ b/magefile.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "log" + "math/rand" "os" "os/exec" "path/filepath" @@ -23,6 +24,8 @@ import ( "sync/atomic" "time" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/elastic/e2e-testing/pkg/downloads" "github.com/elastic/elastic-agent/dev-tools/mage" devtools "github.com/elastic/elastic-agent/dev-tools/mage" @@ -1562,6 +1565,353 @@ func (Integration) Single(ctx context.Context, testName string) error { return integRunner(ctx, false, testName) } +var stateDir = ".integration-cache" +var stateFile = "state.yml" + +// readFrameworkState reads the state file from the integration test framework +func readFrameworkState() (runner.State, error) { + stateFilePath := ".integration-cache/state.yml" + data, err := os.ReadFile(stateFilePath) + if err != nil { + return runner.State{}, fmt.Errorf("could not read state file %q: %w", stateFilePath, err) + } + + state := runner.State{} + if err := yaml.Unmarshal(data, &state); err != nil { + return runner.State{}, fmt.Errorf("failed unmarshal state file %s: %w", stateFilePath, err) + } + + return state, nil +} + +func listVMs() (string, []runner.StateInstance, error) { + builder := strings.Builder{} + state, err := readFrameworkState() + if err != nil { + return "", []runner.StateInstance{}, fmt.Errorf("could not read state file: %w", err) + } + + absStateDir, err := filepath.Abs(stateDir) + if err != nil { + return "", []runner.StateInstance{}, fmt.Errorf("cannot get absolute path from state directory '%s': %w", stateDir, err) + } + + for i, vm := range state.Instances { + isGCP := vm.Provisioner != "multipass" + + t := table.NewWriter() + t.AppendRows([]table.Row{ + {"#", i}, + {"Provisioner", vm.Provisioner}, + {"Name", vm.Name}, + {"ID", vm.ID}, + }) + + if isGCP { + t.AppendRow(table.Row{"Instance ID", vm.Internal["instance_id"]}) + } + + t.AppendRows([]table.Row{ + {"IP", vm.IP}, + {"Private Key", filepath.Join(absStateDir, "id_rsa")}, + {"Public Key", filepath.Join(absStateDir, "id_rsa.pub")}, + {"SSH connection", fmt.Sprintf(`ssh -i %s %s@%s`, filepath.Join(absStateDir, "id_rsa"), vm.Username, vm.IP)}, + }) + + if isGCP { + t.AppendRow(table.Row{"GCP Link", fmt.Sprintf("https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/%s", vm.Internal["instance_id"])}) + } + + builder.WriteString(t.Render()) + builder.WriteString("\n") + } + + return builder.String(), state.Instances, nil +} + +func listStacks() (string, error) { + builder := strings.Builder{} + + state, err := readFrameworkState() + if err != nil { + return "", fmt.Errorf("could not read state file: %w", err) + } + + for i, stack := range state.Stacks { + t := table.NewWriter() + t.AppendRows([]table.Row{ + table.Row{"#", i}, + table.Row{"Type", stack.Provisioner}, + }) + + switch { + case stack.Provisioner == "serverless": + t.AppendRow(table.Row{"Project ID", stack.Internal["deployment_id"]}) + case stack.Provisioner == "stateful": + t.AppendRow(table.Row{"Deployment ID", stack.Internal["deployment_id"]}) + } + t.AppendRows([]table.Row{ + {"Elasticsearch URL", stack.Elasticsearch}, + {"Kibana", stack.Kibana}, + {"Username", stack.Username}, + {"Password", stack.Password}, + }) + builder.WriteString(t.Render()) + builder.WriteString("\n") + } + + return builder.String(), nil +} + +func askForVM() (runner.StateInstance, error) { + vms, instances, err := listVMs() + if err != nil { + fmt.Errorf("cannot list VMs: %w", err) + } + fmt.Fprintf(os.Stderr, vms) + + if len(instances) == 1 { + fmt.Fprintln(os.Stderr, "There is only one VM, auto-selecting it") + return instances[0], nil + } + + id := 0 + fmt.Fprint(os.Stderr, "Instance number: ") + if _, err := fmt.Scanf("%d", &id); err != nil { + return runner.StateInstance{}, fmt.Errorf("could not read instance number: %w:", err) + } + + if id >= len(instances) { + return runner.StateInstance{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(instances)-1) + } + + return instances[id], nil +} + +func askForStack() (runner.Stack, error) { + mg.Deps(Integration.ListStacks) + + state, err := readFrameworkState() + if err != nil { + return runner.Stack{}, fmt.Errorf("could not read state file: %w", err) + } + + if len(state.Stacks) == 1 { + fmt.Println("There is only one Stack, auto-selecting it") + return state.Stacks[0], nil + } + + id := 0 + fmt.Print("Stack number: ") + if _, err := fmt.Scanf("%d", &id); err != nil { + return runner.Stack{}, fmt.Errorf("cannot read Stack number: %w", err) + } + + if id >= len(state.Stacks) { + return runner.Stack{}, fmt.Errorf("Invalid Stack number, it must be between 0 and %d", len(state.Stacks)-1) + } + + return state.Stacks[id], nil +} + +func generateEnvFile(stack runner.Stack) error { + fileExists := true + stat, err := os.Stat("./env.sh") + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("cannot stat 'env.sh': %w", err) + } + fileExists = false + } + + if fileExists { + bkpName := fmt.Sprintf("./env.sh-%d", rand.Int()) + if err := os.Rename(stat.Name(), bkpName); err != nil { + return fmt.Errorf("cannot create backup: %w", err) + } + fmt.Printf("%q already existed, it was moved to %q\n", stat.Name(), bkpName) + } + + f, err := os.Create("./env.sh") + if err != nil { + return fmt.Errorf("Could not create './env.sh': %w", err) + } + defer f.Close() + + fmt.Fprintf(f, "export ELASTICSEARCH_HOST=\"%s\"\n", stack.Elasticsearch) + fmt.Fprintf(f, "export ELASTICSEARCH_USERNAME=\"%s\"\n", stack.Username) + fmt.Fprintf(f, "export ELASTICSEARCH_PASSWORD=\"%s\"\n", stack.Password) + + fmt.Fprintf(f, "export KIBANA_HOST=\"%s\"\n", stack.Kibana) + fmt.Fprintf(f, "export KIBANA_USERNAME=\"%s\"\n", stack.Username) + fmt.Fprintf(f, "export KIBANA_PASSWORD=\"%s\"\n", stack.Password) + + return nil +} + +// PrintState prints details about cloud stacks and VMs +func (Integration) PrintState(ctx context.Context) { + fmt.Println("Virtual Machines") + mg.Deps(Integration.ListVMs) + fmt.Print("\n\n") + fmt.Println("Cloud Stacks") + mg.Deps(Integration.ListStacks) +} + +// ListVMs lists all VMs in a human readable form, including connection details +func (Integration) ListVMs() error { + t, _, err := listVMs() + if err != nil { + fmt.Errorf("cannot list VMs: %w", err) + } + + fmt.Print(t) + + return nil +} + +// SSHVM prints to stdout the SSH command to connect to a VM, a menu is printed to stderr. +func (Integration) SSHVM() error { + absStateDir, err := filepath.Abs(stateDir) + if err != nil { + return fmt.Errorf("cannot get absolute path from state directory '%s': %w", stateDir, err) + } + + vm, err := askForVM() + if err != nil { + fmt.Errorf("cannot get VM: %w", err) + } + + fmt.Println(fmt.Sprintf(`ssh -i %s %s@%s`, filepath.Join(absStateDir, "id_rsa"), vm.Username, vm.IP)) + return nil +} + +// ListStacks lists all stack deployments in a human readable form +func (Integration) ListStacks() error { + stacks, err := listStacks() + if err != nil { + return fmt.Errorf("cannot list stacks: %w", err) + } + + fmt.Print(stacks) + return nil +} + +// GenerateEnvFile generates 'env.sh' containing envvars to connect to a cloud stack +func (Integration) GenerateEnvFile() error { + stack, err := askForStack() + if err != nil { + return fmt.Errorf("cannot get stack: %w", err) + } + + if err := generateEnvFile(stack); err != nil { + return fmt.Errorf("cannot generate env file: %w", err) + } + fmt.Println("run 'source ./env.sh' to load the environment variables to your shell") + + return nil +} + +// DeployEnvFile generates and deploys to a VM 'env.sh' containing envvars to connect to a cloud stack +func (Integration) DeployEnvFile() error { + stack, err := askForStack() + if err != nil { + return fmt.Errorf("cannot get stack: %w", err) + } + + if err := generateEnvFile(stack); err != nil { + return fmt.Errorf("cannot generate env file: %w", err) + } + + fullEnvFilepath, err := filepath.Abs("./env.sh") + if err != nil { + return fmt.Errorf("cannot get full filepath for env file: %w", err) + } + + absStateDir, err := filepath.Abs(stateDir) + if err != nil { + return fmt.Errorf("cannot get absolute path from state directory '%s': %w", stateDir, err) + } + keyFile := filepath.Join(absStateDir, "id_rsa") + + vm, err := askForVM() + if err != nil { + return fmt.Errorf("cannot get VM: %w", err) + } + + cmd := exec.Command("scp", "-i", keyFile, fullEnvFilepath, fmt.Sprintf("%s@%s:~/env.sh", vm.Username, vm.IP)) + if err := cmd.Run(); err != nil { + return fmt.Errorf("could not copy env file to VM: %w", err) + } + + return nil +} + +// DeployDebugTools installs all necessary tools to debug tests from a VM +func (Integration) DeployDebugTools() error { + absStateDir, err := filepath.Abs(stateDir) + if err != nil { + return fmt.Errorf("cannot get absolute path from state directory '%s': %w", stateDir, err) + } + keyFile := filepath.Join(absStateDir, "id_rsa") + + vm, err := askForVM() + if err != nil { + return fmt.Errorf("cannot get VM: %w", err) + } + + isWindowsVM := strings.Contains(vm.ID, "windows") + + commands := []string{ + fmt.Sprintf("sudo chown -R %s:%s $HOME/go/pkg", vm.Username, vm.Username), + "go install github.com/go-delve/delve/cmd/dlv@latest", + } + + if isWindowsVM { + commands = append(commands, + "choco install -y git", + "if exist mage rmdir /s /q mage", + "if exist elastic-agent rmdir /s /q elastic-agent", + ) + } else { + commands = append(commands, + `echo 'export PATH=$PATH:'"$HOME/go/bin" |sudo tee /root/.bashrc`, + "rm -rf mage", + "rm -rf elastic-agent", + "sudo apt install -y docker.io", + "sudo systemctl enable --now docker", + "sudo usermod -aG docker $USER", + ) + } + + commands = append(commands, + "git clone https://github.com/magefile/mage", + "cd mage && go run bootstrap.go", + "git clone https://github.com/elastic/elastic-agent", + ) + + if isWindowsVM { + commands = append(commands, "cd elastic-agent && xcopy /s /e /y ..\\agent\\ .\\") + } else { + commands = append(commands, "cd elastic-agent && cp -r ~/agent/* ./") + } + + for _, c := range commands { + cmd := exec.Command("ssh", "-i", keyFile, fmt.Sprintf("%s@%s", vm.Username, vm.IP), c) + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + if err := cmd.Run(); err != nil { + return err + } + } + + fmt.Println("Delve, Mage have been installed and added to the path") + fmt.Println("~/elastic-agent") + return nil +} + // PrepareOnRemote shouldn't be called locally (called on remote host to prepare it for testing) func (Integration) PrepareOnRemote() { mg.Deps(mage.InstallGoTestTools)