Skip to content

Commit

Permalink
tests(e2e): collect and upload e2e test logs on failure (#942)
Browse files Browse the repository at this point in the history 
In case we get an error during the tests, attempt to collect some
potentially relevant logs from the remote client and download them to
`logs/$HOSTNAME`.

We currently collect apport logs, journalctl, and an archive of
`/var/log` (excluding `/var/log/journal`). They are uploaded as GH
actions artifacts only if tests fail.

Example: https://github.com/GabrielNagy/adsys/actions/runs/8205179603

Fixes UDENG-1537
  • Loading branch information
@GabrielNagy
GabrielNagy authored Mar 11, 2024
2 parents a6e8fbb + e106432 commit bafdd5f
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 42 deletions.
6 changes: 6 additions & 0 deletions .github/workflows/e2e-tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,12 @@ jobs:
run: go run ./e2e/cmd/run_tests/04_test_pro_managers
- name: 'Test: PAM and Kerberos ticket cache'
run: go run ./e2e/cmd/run_tests/05_test_pam_krb5cc
- name: Collect logs on failure
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: e2e-logs-${{ matrix.codename }}
path: logs/
- name: Deprovision resources
if: ${{ always() }}
run: |
Expand Down
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ debian/adsys-windows
# E2E testing
inventory.yaml
e2e/assets/gpo/**/*.pol
logs/

# GitHub CI temporary files
node_modules
Expand Down
21 changes: 12 additions & 9 deletions e2e/cmd/run_tests/03_test_non_pro_managers/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,38 +54,41 @@ func validate(_ context.Context, cmd *command.Command) (err error) {
return nil
}

func action(ctx context.Context, cmd *command.Command) error {
client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
func action(ctx context.Context, cmd *command.Command) (err error) {
rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
if err != nil {
return fmt.Errorf("failed to connect to VM: %w", err)
}

//nolint:errcheck // This is a best effort to collect logs
defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname)

// Reboot machine to apply machine policies
if err := client.Reboot(); err != nil {
if err := rootClient.Reboot(); err != nil {
return err
}

// Assert machine policies were applied
if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-format", "'12h'"); err != nil {
if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-format", "'12h'"); err != nil {
return err
}
if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-show-weekday", "false"); err != nil {
if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/desktop/interface/clock-show-weekday", "false"); err != nil {
return err
}
if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-enable", "true"); err != nil {
if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-enable", "true"); err != nil {
return err
}
if err := client.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-text", "'Sample banner text'"); err != nil {
if err := rootClient.RequireEqual(ctx, "DCONF_PROFILE=gdm dconf read /org/gnome/login-screen/banner-message-text", "'Sample banner text'"); err != nil {
return err
}

// Pro policies should not be applied yet
if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "''"); err != nil {
if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "''"); err != nil {
return err
}

// Assert user GPO policies were applied
client, err = remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword)
client, err := remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword)
if err != nil {
return fmt.Errorf("failed to connect to VM: %w", err)
}
Expand Down
62 changes: 30 additions & 32 deletions e2e/cmd/run_tests/04_test_pro_managers/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,49 +70,47 @@ func validate(_ context.Context, cmd *command.Command) (err error) {
return nil
}

func action(ctx context.Context, cmd *command.Command) error {
client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
func action(ctx context.Context, cmd *command.Command) (err error) {
rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
if err != nil {
return fmt.Errorf("failed to connect to VM: %w", err)
}

defer func() {
client, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
if err != nil {
log.Errorf("Teardown: Failed to connect to VM as root: %v", err)
}
//nolint:errcheck // This is a best effort to collect logs
defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname)

if _, err := client.Run(ctx, "adsysctl policy purge --all -v"); err != nil {
defer func() {
if _, err := rootClient.Run(ctx, "adsysctl policy purge --all -v"); err != nil {
log.Errorf("Teardown: Failed to purge policies: %v", err)
}

// Detach client from Ubuntu Pro
if _, err := client.Run(ctx, "yes | pro detach"); err != nil {
if _, err := rootClient.Run(ctx, "yes | pro detach"); err != nil {
log.Errorf("Teardown: Failed to detach client from Ubuntu Pro: %v", err)
}

// Remove some files generated by the scripts policy to ensure idempotency
if _, err := client.Run(ctx, "systemctl stop adsys-machine-scripts"); err != nil {
if _, err := rootClient.Run(ctx, "systemctl stop adsys-machine-scripts"); err != nil {
log.Errorf("Teardown: Failed to stop machine scripts service: %v", err)
}
if _, err := client.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script /etc/created-by-adsys-machine-shutdown-script"); err != nil {
if _, err := rootClient.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script /etc/created-by-adsys-machine-shutdown-script"); err != nil {
log.Errorf("Teardown: Failed to remove machine startup/shutdown scripts: %v", err)
}
if _, err := client.Run(ctx, "rm -f /home/*/created-by-adsys-admin-logon-script /home/*/created-by-adsys-admin-logoff-script /home/*/created-by-adsys-user-logon-script"); err != nil {
if _, err := rootClient.Run(ctx, "rm -f /home/*/created-by-adsys-admin-logon-script /home/*/created-by-adsys-admin-logoff-script /home/*/created-by-adsys-user-logon-script"); err != nil {
log.Errorf("Teardown: Failed to remove user scripts: %v", err)
}
}()

// Attach client to Ubuntu Pro
if _, err := client.Run(ctx, fmt.Sprintf("pro attach %s --no-auto-enable", proToken)); err != nil {
if _, err := rootClient.Run(ctx, fmt.Sprintf("pro attach %s --no-auto-enable", proToken)); err != nil {
return fmt.Errorf("failed to attach client to Ubuntu Pro: %w", err)
}
if err := client.RequireContains(ctx, "pro status --wait", "Subscription: Ubuntu Pro"); err != nil {
if err := rootClient.RequireContains(ctx, "pro status --wait", "Subscription: Ubuntu Pro"); err != nil {
return fmt.Errorf("failed to confirm client is attached to Ubuntu Pro: %w", err)
}

// Start timer-triggered service to update policies
if _, err := client.Run(ctx, "systemctl restart adsys-gpo-refresh"); err != nil {
if _, err := rootClient.Run(ctx, "systemctl restart adsys-gpo-refresh"); err != nil {
// cifs mount on focal asks for a password via systemd-ask-password,
// which we cannot interact with in the context of a script...
// Assume everything went fine since we don't care about system mounts
Expand All @@ -125,44 +123,44 @@ func action(ctx context.Context, cmd *command.Command) error {
// Assert machine policies were applied
/// Mounts
if cmd.Inventory.Codename != "focal" { // mount behavior is spotty on focal so avoid asserting it
if err := client.RequireEqual(ctx, "cat /adsys/nfs/warthogs.biz/system-mount-nfs/file.txt", expectedMountedFileContents); err != nil {
if err := rootClient.RequireEqual(ctx, "cat /adsys/nfs/warthogs.biz/system-mount-nfs/file.txt", expectedMountedFileContents); err != nil {
return err
}
if err := client.RequireEqual(ctx, "cat /adsys/cifs/warthogs.biz/system-mount-smb/file.txt", expectedMountedFileContents); err != nil {
if err := rootClient.RequireEqual(ctx, "cat /adsys/cifs/warthogs.biz/system-mount-smb/file.txt", expectedMountedFileContents); err != nil {
return err
}
}

/// Privilege escalation
if err := client.RequireEqual(ctx, "cat /etc/sudoers.d/99-adsys-privilege-enforcement", `# This file is managed by adsys.
if err := rootClient.RequireEqual(ctx, "cat /etc/sudoers.d/99-adsys-privilege-enforcement", `# This file is managed by adsys.
# Do not edit this file manually.
# Any changes will be overwritten.
"adminuser@warthogs.biz" ALL=(ALL:ALL) ALL`); err != nil {
return err
}
// Only partly assert the polkit file contents as there are differences in polkit configurations between Ubuntu versions
if err := client.RequireContains(ctx, "cat /etc/polkit-1/localauthority.conf.d/99-adsys-privilege-enforcement.conf", "unix-user:adminuser@warthogs.biz"); err != nil {
if err := rootClient.RequireContains(ctx, "cat /etc/polkit-1/localauthority.conf.d/99-adsys-privilege-enforcement.conf", "unix-user:adminuser@warthogs.biz"); err != nil {
return err
}

/// AppArmor
if cmd.Inventory.Codename != "focal" { // aa-status raises a Python exception on focal
if err := client.RequireContains(ctx, "aa-status", "/usr/bin/foo=(complain)"); err != nil {
if err := rootClient.RequireContains(ctx, "aa-status", "/usr/bin/foo=(complain)"); err != nil {
return err
}
}

/// Scripts
if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil {
if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil {
return fmt.Errorf("%w: file should have been created by the adsys-gpo-refresh service", err)
}
// Remove startup script so we can check creation at next reboot
if _, err := client.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script"); err != nil {
if _, err := rootClient.Run(ctx, "rm -f /etc/created-by-adsys-machine-startup-script"); err != nil {
log.Errorf("Failed to remove machine startup scripts: %v", err)
}

if err := client.RequireNoFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil {
if err := rootClient.RequireNoFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil {
return err
}

Expand All @@ -171,46 +169,46 @@ func action(ctx context.Context, cmd *command.Command) error {
/// Certificates
// Enrollment takes a few seconds, so no better way to do this than an arbitrary sleep :)
time.Sleep(5 * time.Second)
if err := client.RequireContains(ctx, "getcert list -i warthogs-CA.Machine", "status: MONITORING"); err != nil {
if err := rootClient.RequireContains(ctx, "getcert list -i warthogs-CA.Machine", "status: MONITORING"); err != nil {
return err
}

/// Proxy
if err := client.RequireEqual(ctx, "cat /etc/apt/apt.conf.d/99ubuntu-proxy-manager", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten
if err := rootClient.RequireEqual(ctx, "cat /etc/apt/apt.conf.d/99ubuntu-proxy-manager", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten
Acquire::ftp::Proxy "http://127.0.0.1:8080";`); err != nil {
return err
}
if err := client.RequireEqual(ctx, "cat /etc/environment.d/99ubuntu-proxy-manager.conf", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten
if err := rootClient.RequireEqual(ctx, "cat /etc/environment.d/99ubuntu-proxy-manager.conf", `### This file was generated by ubuntu-proxy-manager - manual changes will be overwritten
FTP_PROXY="http://127.0.0.1:8080"
ftp_proxy="http://127.0.0.1:8080"`); err != nil {
return err
}
if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "'127.0.0.1'"); err != nil {
if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp host", "'127.0.0.1'"); err != nil {
return err
}

if err := client.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp port", "8080"); err != nil {
if err := rootClient.RequireEqual(ctx, "gsettings get org.gnome.system.proxy.ftp port", "8080"); err != nil {
return err
}
}

// Reboot and check machine scripts
if err := client.Reboot(); err != nil {
if err := rootClient.Reboot(); err != nil {
return err
}

// Sleep a few seconds to ensure the machine startup script has time to run
time.Sleep(10 * time.Second)

if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil {
if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-shutdown-script"); err != nil {
return err
}
if err := client.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil {
if err := rootClient.RequireFileExists(ctx, "/etc/created-by-adsys-machine-startup-script"); err != nil {
return err
}

////// Start policies for $HOST-USR@WARTHOGS.BIZ
client, err = remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword)
client, err := remote.NewClient(cmd.Inventory.IP, fmt.Sprintf("%s-usr@warthogs.biz", cmd.Inventory.Hostname), remote.DomainUserPassword)
if err != nil {
return fmt.Errorf("failed to connect to VM: %w", err)
}
Expand Down
5 changes: 4 additions & 1 deletion e2e/cmd/run_tests/05_test_pam_krb5cc/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,15 @@ func validate(_ context.Context, cmd *command.Command) (err error) {
return nil
}

func action(ctx context.Context, cmd *command.Command) error {
func action(ctx context.Context, cmd *command.Command) (err error) {
rootClient, err := remote.NewClient(cmd.Inventory.IP, "root", sshKey)
if err != nil {
return fmt.Errorf("failed to connect to VM: %w", err)
}

//nolint:errcheck // This is a best effort to collect logs
defer rootClient.CollectLogsOnFailure(ctx, &err, cmd.Inventory.Hostname)

defer func() {
if _, err := rootClient.Run(ctx, "rm -f /etc/adsys.yaml"); err != nil {
log.Errorf("Teardown: Failed to remove adsys configuration file: %v", err)
Expand Down
94 changes: 94 additions & 0 deletions e2e/internal/remote/remote.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,36 @@ func (c Client) Upload(localPath string, remotePath string) error {
return nil
}

// Download downloads the given remote file to the local path.
func (c Client) Download(remotePath string, localPath string) error {
log.Infof("Downloading %q from host %q to %q", remotePath, c.client.RemoteAddr(), localPath)

ftp, err := sftp.NewClient(c.client)
if err != nil {
return err
}
defer ftp.Close()

remote, err := ftp.Open(remotePath)
if err != nil {
return err
}
defer remote.Close()

local, err := os.Create(localPath)
if err != nil {
return err
}
defer local.Close()

if _, err := remote.WriteTo(local); err != nil {
return err
}
log.Info("File downloaded successfully")

return nil
}

// Reboot reboots the remote host and waits for it to come back online, then
// reestablishes the SSH connection.
// It first waits for the host to go offline, then returns an error if the host
Expand Down Expand Up @@ -272,3 +302,67 @@ func (c *Client) Reboot() error {
}
}
}

// CollectLogs collects logs from the remote host and writes them to disk under
// a relative logs directory named after the client host.
func (c *Client) CollectLogs(ctx context.Context, hostname string) (err error) {
defer func() {
if err != nil {
log.Errorf("Failed to collect logs from host %q: %v", hostname, err)
}
}()

log.Infof("Collecting logs from host %q", c.client.RemoteAddr().String())

// Create local directory to store logs
logDir := filepath.Join("logs", hostname)
if err := os.MkdirAll(logDir, 0700); err != nil {
return fmt.Errorf("failed to create log directory: %w", err)
}

// Check if we are still connected to remote server, attempt to reconnect if not
if c.client == nil {
c.client, err = ssh.Dial("tcp", c.host+":22", c.config)
if err != nil {
return fmt.Errorf("failed to reconnect to %q: %w", c.host, err)
}
}

// Run ubuntu-bug to collect logs
_, err = c.Run(ctx, "APPORT_DISABLE_DISTRO_CHECK=1 ubuntu-bug --save=/root/bug adsys")
if err != nil {
return fmt.Errorf("failed to collect logs: %w", err)
}
// Save journalctl logs
_, err = c.Run(ctx, "journalctl --no-pager --output=short-precise --no-hostname > /root/journal")
if err != nil {
return fmt.Errorf("failed to read logs: %w", err)
}

// Archive and download /var/log
if _, err := c.Run(ctx, "tar --exclude=/var/log/journal -czf /root/varlog.tar.gz /var/log"); err != nil {
return fmt.Errorf("failed to archive logs: %w", err)
}

// Download remote logs
if err := c.Download("/root/varlog.tar.gz", filepath.Join(logDir, "varlog.tar.gz")); err != nil {
return fmt.Errorf("failed to download logs: %w", err)
}
if err := c.Download("/root/bug", filepath.Join(logDir, "apport.log")); err != nil {
return fmt.Errorf("failed to download logs: %w", err)
}
if err := c.Download("/root/journal", filepath.Join(logDir, "journal.log")); err != nil {
return fmt.Errorf("failed to download logs: %w", err)
}

return nil
}

// CollectLogsOnFailure collects logs from the remote host and writes them to disk if passed a non-nil error.
func (c *Client) CollectLogsOnFailure(ctx context.Context, err *error, hostname string) error {
if *err != nil {
return c.CollectLogs(ctx, hostname)
}

return nil
}

0 comments on commit bafdd5f

Please sign in to comment.