diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml index 9b97574f0..c9ff2b442 100644 --- a/.github/workflows/e2e-tests.yaml +++ b/.github/workflows/e2e-tests.yaml @@ -114,6 +114,12 @@ jobs: run: go run ./e2e/cmd/run_tests/04_test_pro_managers - name: 'Test: PAM and Kerberos ticket cache' run: go run ./e2e/cmd/run_tests/05_test_pam_krb5cc + - name: Collect logs on failure + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: e2e-logs-${{ matrix.codename }} + path: logs/ - name: Deprovision resources if: ${{ always() }} run: | diff --git a/.gitignore b/.gitignore index c7f798cd1..a420a4354 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ debian/adsys-windows # E2E testing inventory.yaml e2e/assets/gpo/**/*.pol +logs/ # GitHub CI temporary files node_modules diff --git a/e2e/cmd/run_tests/03_test_non_pro_managers/main.go b/e2e/cmd/run_tests/03_test_non_pro_managers/main.go index 49b114f48..44b01932e 100644 --- a/e2e/cmd/run_tests/03_test_non_pro_managers/main.go +++ b/e2e/cmd/run_tests/03_test_non_pro_managers/main.go @@ -54,38 +54,41 @@ func validate(_ context.Context, cmd *command.Command) (err error) { return nil } -func action(ctx context.Context, cmd *command.Command) error { - client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) +func action(ctx context.Context, cmd *command.Command) (err error) { + rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) if err != nil { return fmt.Errorf("failed to connect to VM: %w", err) } + //nolint:errcheck // This is a best effort to collect logs + defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname) + // Reboot machine to apply machine policies - if err := client.Reboot(); err != nil { + if err := rootClient.Reboot(); err != nil { return err } // Assert machine policies were applied - if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-format", "'12h'"); err != nil { + if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-format", "'12h'"); err != nil { return err } - if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-show-weekday", "false"); err != nil { + if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-show-weekday", "false"); err != nil { return err } - if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-enable", "true"); err != nil { + if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-enable", "true"); err != nil { return err } - if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-text", "'Sample banner text'"); err != nil { + if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-text", "'Sample banner text'"); err != nil { return err } // Pro policies should not be applied yet - if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "''"); err != nil { + if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "''"); err != nil { return err } // Assert user GPO policies were applied - client, err = remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword) + client, err := remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword) if err != nil { return fmt.Errorf("failed to connect to VM: %w", err) } diff --git a/e2e/cmd/run_tests/04_test_pro_managers/main.go b/e2e/cmd/run_tests/04_test_pro_managers/main.go index d6f423699..35a1e587a 100644 --- a/e2e/cmd/run_tests/04_test_pro_managers/main.go +++ b/e2e/cmd/run_tests/04_test_pro_managers/main.go @@ -70,49 +70,47 @@ func validate(_ context.Context, cmd *command.Command) (err error) { return nil } -func action(ctx context.Context, cmd *command.Command) error { - client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) +func action(ctx context.Context, cmd *command.Command) (err error) { + rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) if err != nil { return fmt.Errorf("failed to connect to VM: %w", err) } - defer func() { - client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) - if err != nil { - log.Errorf("Teardown: Failed to connect to VM as root: %v", err) - } + //nolint:errcheck // This is a best effort to collect logs + defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname) - if _, err := client.Run(ctx, "adsysctl policy purge --all -v"); err != nil { + defer func() { + if _, err := rootClient.Run(ctx, "adsysctl policy purge --all -v"); err != nil { log.Errorf("Teardown: Failed to purge policies: %v", err) } // Detach client from Ubuntu Pro - if _, err := client.Run(ctx, "yes | pro detach"); err != nil { + if _, err := rootClient.Run(ctx, "yes | pro detach"); err != nil { log.Errorf("Teardown: Failed to detach client from Ubuntu Pro: %v", err) } // Remove some files generated by the scripts policy to ensure idempotency - if _, err := client.Run(ctx, "systemctl stop adsys-machine-scripts"); err != nil { + if _, err := rootClient.Run(ctx, "systemctl stop adsys-machine-scripts"); err != nil { log.Errorf("Teardown: Failed to stop machine scripts service: %v", err) } - if _, err := client.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script /etc/created-by-adsys-machine-shutdown-script"); err != nil { + if _, err := rootClient.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script /etc/created-by-adsys-machine-shutdown-script"); err != nil { log.Errorf("Teardown: Failed to remove machine startup/shutdown scripts: %v", err) } - if _, err := client.Run(ctx, "rm -f /home/*/created-by-adsys-admin-logon-script /home/*/created-by-adsys-admin-logoff-script /home/*/created-by-adsys-user-logon-script"); err != nil { + if _, err := rootClient.Run(ctx, "rm -f /home/*/created-by-adsys-admin-logon-script /home/*/created-by-adsys-admin-logoff-script /home/*/created-by-adsys-user-logon-script"); err != nil { log.Errorf("Teardown: Failed to remove user scripts: %v", err) } }() // Attach client to Ubuntu Pro - if _, err := client.Run(ctx, fmt.Sprintf("pro attach %s --no-auto-enable", proToken)); err != nil { + if _, err := rootClient.Run(ctx, fmt.Sprintf("pro attach %s --no-auto-enable", proToken)); err != nil { return fmt.Errorf("failed to attach client to Ubuntu Pro: %w", err) } - if err := client.RequireContains(ctx, "pro status --wait", "Subscription: Ubuntu Pro"); err != nil { + if err := rootClient.RequireContains(ctx, "pro status --wait", "Subscription: Ubuntu Pro"); err != nil { return fmt.Errorf("failed to confirm client is attached to Ubuntu Pro: %w", err) } // Start timer-triggered service to update policies - if _, err := client.Run(ctx, "systemctl restart adsys-gpo-refresh"); err != nil { + if _, err := rootClient.Run(ctx, "systemctl restart adsys-gpo-refresh"); err != nil { // cifs mount on focal asks for a password via systemd-ask-password, // which we cannot interact with in the context of a script... // Assume everything went fine since we don't care about system mounts @@ -125,16 +123,16 @@ func action(ctx context.Context, cmd *command.Command) error { // Assert machine policies were applied /// Mounts if cmd.Inventory.Codename != "focal" { // mount behavior is spotty on focal so avoid asserting it - if err := client.RequireEqual(ctx, "cat /adsys/nfs/warthogs.biz/system-mount-nfs/file.txt", expectedMountedFileContents); err != nil { + if err := rootClient.RequireEqual(ctx, "cat /adsys/nfs/warthogs.biz/system-mount-nfs/file.txt", expectedMountedFileContents); err != nil { return err } - if err := client.RequireEqual(ctx, "cat /adsys/cifs/warthogs.biz/system-mount-smb/file.txt", expectedMountedFileContents); err != nil { + if err := rootClient.RequireEqual(ctx, "cat /adsys/cifs/warthogs.biz/system-mount-smb/file.txt", expectedMountedFileContents); err != nil { return err } } /// Privilege escalation - if err := client.RequireEqual(ctx, "cat /etc/sudoers.d/99-adsys-privilege-enforcement", `# This file is managed by adsys. + if err := rootClient.RequireEqual(ctx, "cat /etc/sudoers.d/99-adsys-privilege-enforcement", `# This file is managed by adsys. # Do not edit this file manually. # Any changes will be overwritten. @@ -142,27 +140,27 @@ func action(ctx context.Context, cmd *command.Command) error { return err } // Only partly assert the polkit file contents as there are differences in polkit configurations between Ubuntu versions - if err := client.RequireContains(ctx, "cat /etc/polkit-1/localauthority.conf.d/99-adsys-privilege-enforcement.conf", "unix-user:adminuser@warthogs.biz"); err != nil { + if err := rootClient.RequireContains(ctx, "cat /etc/polkit-1/localauthority.conf.d/99-adsys-privilege-enforcement.conf", "unix-user:adminuser@warthogs.biz"); err != nil { return err } /// AppArmor if cmd.Inventory.Codename != "focal" { // aa-status raises a Python exception on focal - if err := client.RequireContains(ctx, "aa-status", "/usr/bin/foo=(complain)"); err != nil { + if err := rootClient.RequireContains(ctx, "aa-status", "/usr/bin/foo=(complain)"); err != nil { return err } } /// Scripts - if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil { + if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil { return fmt.Errorf("%w: file should have been created by the adsys-gpo-refresh service", err) } // Remove startup script so we can check creation at next reboot - if _, err := client.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script"); err != nil { + if _, err := rootClient.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script"); err != nil { log.Errorf("Failed to remove machine startup scripts: %v", err) } - if err := client.RequireNoFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil { + if err := rootClient.RequireNoFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil { return err } @@ -171,46 +169,46 @@ func action(ctx context.Context, cmd *command.Command) error { /// Certificates // Enrollment takes a few seconds, so no better way to do this than an arbitrary sleep :) time.Sleep(5 * time.Second) - if err := client.RequireContains(ctx, "getcert list -i warthogs-CA.Machine", "status: MONITORING"); err != nil { + if err := rootClient.RequireContains(ctx, "getcert list -i warthogs-CA.Machine", "status: MONITORING"); err != nil { return err } /// Proxy - if err := client.RequireEqual(ctx, "cat /etc/apt/apt.conf.d/99ubuntu-proxy-manager", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten + if err := rootClient.RequireEqual(ctx, "cat /etc/apt/apt.conf.d/99ubuntu-proxy-manager", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten Acquire::ftp::Proxy "http://127.0.0.1:8080";`); err != nil { return err } - if err := client.RequireEqual(ctx, "cat /etc/environment.d/99ubuntu-proxy-manager.conf", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten + if err := rootClient.RequireEqual(ctx, "cat /etc/environment.d/99ubuntu-proxy-manager.conf", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten FTP_PROXY="http://127.0.0.1:8080" ftp_proxy="http://127.0.0.1:8080"`); err != nil { return err } - if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "'127.0.0.1'"); err != nil { + if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "'127.0.0.1'"); err != nil { return err } - if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp port", "8080"); err != nil { + if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp port", "8080"); err != nil { return err } } // Reboot and check machine scripts - if err := client.Reboot(); err != nil { + if err := rootClient.Reboot(); err != nil { return err } // Sleep a few seconds to ensure the machine startup script has time to run time.Sleep(10 * time.Second) - if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil { + if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil { return err } - if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil { + if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil { return err } ////// Start policies for $HOST-USR@WARTHOGS.BIZ - client, err = remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword) + client, err := remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword) if err != nil { return fmt.Errorf("failed to connect to VM: %w", err) } diff --git a/e2e/cmd/run_tests/05_test_pam_krb5cc/main.go b/e2e/cmd/run_tests/05_test_pam_krb5cc/main.go index 887208fef..35a7506fc 100644 --- a/e2e/cmd/run_tests/05_test_pam_krb5cc/main.go +++ b/e2e/cmd/run_tests/05_test_pam_krb5cc/main.go @@ -43,12 +43,15 @@ func validate(_ context.Context, cmd *command.Command) (err error) { return nil } -func action(ctx context.Context, cmd *command.Command) error { +func action(ctx context.Context, cmd *command.Command) (err error) { rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey) if err != nil { return fmt.Errorf("failed to connect to VM: %w", err) } + //nolint:errcheck // This is a best effort to collect logs + defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname) + defer func() { if _, err := rootClient.Run(ctx, "rm -f /etc/adsys.yaml"); err != nil { log.Errorf("Teardown: Failed to remove adsys configuration file: %v", err) diff --git a/e2e/internal/remote/remote.go b/e2e/internal/remote/remote.go index 593f5d8a6..ef142fa81 100644 --- a/e2e/internal/remote/remote.go +++ b/e2e/internal/remote/remote.go @@ -228,6 +228,36 @@ func (c Client) Upload(localPath string, remotePath string) error { return nil } +// Download downloads the given remote file to the local path. +func (c Client) Download(remotePath string, localPath string) error { + log.Infof("Downloading %q from host %q to %q", remotePath, c.client.RemoteAddr(), localPath) + + ftp, err := sftp.NewClient(c.client) + if err != nil { + return err + } + defer ftp.Close() + + remote, err := ftp.Open(remotePath) + if err != nil { + return err + } + defer remote.Close() + + local, err := os.Create(localPath) + if err != nil { + return err + } + defer local.Close() + + if _, err := remote.WriteTo(local); err != nil { + return err + } + log.Info("File downloaded successfully") + + return nil +} + // Reboot reboots the remote host and waits for it to come back online, then // reestablishes the SSH connection. // It first waits for the host to go offline, then returns an error if the host @@ -272,3 +302,67 @@ func (c *Client) Reboot() error { } } } + +// CollectLogs collects logs from the remote host and writes them to disk under +// a relative logs directory named after the client host. +func (c *Client) CollectLogs(ctx context.Context, hostname string) (err error) { + defer func() { + if err != nil { + log.Errorf("Failed to collect logs from host %q: %v", hostname, err) + } + }() + + log.Infof("Collecting logs from host %q", c.client.RemoteAddr().String()) + + // Create local directory to store logs + logDir := filepath.Join("logs", hostname) + if err := os.MkdirAll(logDir, 0700); err != nil { + return fmt.Errorf("failed to create log directory: %w", err) + } + + // Check if we are still connected to remote server, attempt to reconnect if not + if c.client == nil { + c.client, err = ssh.Dial("tcp", c.host+":22", c.config) + if err != nil { + return fmt.Errorf("failed to reconnect to %q: %w", c.host, err) + } + } + + // Run ubuntu-bug to collect logs + _, err = c.Run(ctx, "APPORT_DISABLE_DISTRO_CHECK=1 ubuntu-bug --save=/root/bug adsys") + if err != nil { + return fmt.Errorf("failed to collect logs: %w", err) + } + // Save journalctl logs + _, err = c.Run(ctx, "journalctl --no-pager --output=short-precise --no-hostname > /root/journal") + if err != nil { + return fmt.Errorf("failed to read logs: %w", err) + } + + // Archive and download /var/log + if _, err := c.Run(ctx, "tar --exclude=/var/log/journal -czf /root/varlog.tar.gz /var/log"); err != nil { + return fmt.Errorf("failed to archive logs: %w", err) + } + + // Download remote logs + if err := c.Download("/root/varlog.tar.gz", filepath.Join(logDir, "varlog.tar.gz")); err != nil { + return fmt.Errorf("failed to download logs: %w", err) + } + if err := c.Download("/root/bug", filepath.Join(logDir, "apport.log")); err != nil { + return fmt.Errorf("failed to download logs: %w", err) + } + if err := c.Download("/root/journal", filepath.Join(logDir, "journal.log")); err != nil { + return fmt.Errorf("failed to download logs: %w", err) + } + + return nil +} + +// CollectLogsOnFailure collects logs from the remote host and writes them to disk if passed a non-nil error. +func (c *Client) CollectLogsOnFailure(ctx context.Context, err *error, hostname string) error { + if *err != nil { + return c.CollectLogs(ctx, hostname) + } + + return nil +}