test: add end to end tests (#203)

* test: add end to end tests

* add test script

* add complete test
delete resources across all zones
add new asserts

* add more asserts

* Use go tests instead of a main

* run packer from go tests and merge *_test.go with packer files

* add cassettes to test checks

* add cassettes to packer

* fix execution with cassettes

* improve doc

* add cassettes

* run e2e tests in CI

* fix tests in CI

* lint

* split test and e2e_test in makefile

* add e2e test to ci

* skip broken test

.github/workflows/unit-tests.yml

@@ -30,6 +30,8 @@ jobs:
         run: make build
       - name: Run unit tests
         run: make test
+      - name: Run e2e tests
+        run: make e2e_test
 
   generate:
     strategy:

GNUmakefile

@@ -3,7 +3,8 @@ BINARY=packer-plugin-${NAME}
 PLUGIN_FQN="$(shell grep -E '^module' <go.mod | sed -E 's/module *//')"
 
 COUNT?=1
-TEST?=$(shell go list ./...)
+E2E_TEST?=$(shell go list ./internal/tests)
+TEST?=$(filter-out $(E2E_TEST),$(shell go list ./...))
 HASHICORP_PACKER_PLUGIN_SDK_VERSION?=$(shell go list -m github.com/hashicorp/packer-plugin-sdk | cut -d " " -f2)
 
 .PHONY: dev
@@ -18,6 +19,9 @@ dev:
 test:
 	@go test -race -count $(COUNT) $(TEST) -timeout=3m
 
+e2e_test:
+	make -C e2e_tests test
+
 install-packer-sdc: ## Install packer sofware development command
 	go install github.com/hashicorp/packer-plugin-sdk/cmd/packer-sdc@${HASHICORP_PACKER_PLUGIN_SDK_VERSION}
 

builder/scaleway/builder.go

@@ -9,12 +9,14 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer-plugin-sdk/communicator"
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
 	"github.com/hashicorp/packer-plugin-sdk/multistep/commonsteps"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"github.com/scaleway/packer-plugin-scaleway/internal/vcr"
 	"github.com/scaleway/scaleway-sdk-go/scw"
 )
 
@@ -57,6 +59,19 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		clientOpts = append(clientOpts, scw.WithAPIURL(b.Config.APIURL))
 	}
 
+	// Only use cassette if vcr.UpdateCassettesEnvVariable env variable is used.
+	// It must at least be set to false when wanting to use local cassettes.
+	if _, isSet := os.LookupEnv(vcr.UpdateCassettesEnvVariable); isSet {
+		client, cleanup, err := vcr.GetHTTPRecorder(filepath.Join("testdata", b.Config.ImageName+".cassette"), vcr.UpdateCassettes)
+		if err != nil {
+			ui.Error(err.Error())
+			return nil, err
+		}
+		defer cleanup()
+
+		clientOpts = append(clientOpts, scw.WithHTTPClient(client))
+	}
+
 	client, err := scw.NewClient(clientOpts...)
 	if err != nil {
 		ui.Error(err.Error())

e2e_tests/GNUmakefile

@@ -0,0 +1,8 @@
+build-binary:
+	make -C .. build
+
+install-plugin:
+	packer plugins install -path ../packer-plugin-scaleway "github.com/scaleway/scaleway"
+
+test: build-binary install-plugin
+	./test.sh

e2e_tests/README.md

@@ -0,0 +1,24 @@
+# End-to-end tests
+
+This folder contains scripts and a makefile to help run end to end tests.
+
+## Write a test
+
+Create a new test in `../internal/tests`.
+
+## Cassettes
+
+To run easily in a CI, tests can be run while recording http requests. This allows pipelines to test without token by using recorded requests.
+
+- To record cassettes, you must set `PACKER_UPDATE_CASSETTES=true`.
+- To use recorded cassettes, you must set `PACKER_UPDATE_CASSETTES=false`
+
+## Running tests
+
+`PACKER_UPDATE_CASSETTES=true make test`
+
+Test script will create a new project for you then run all tests before cleaning up the project
+
+## Test environment
+
+Tests should run in an empty project. Tests will check for non deleted resources and a non-empty project will interfere with the checks.

e2e_tests/clean.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+## This script will clean a project and its resources.
+
+if [ "$#" -ne 1 ]
+then
+  echo "Usage: $0 [project-id]"
+  exit 1
+fi
+
+export SCW_DEFAULT_PROJECT_ID=$1
+
+# Clean images
+scw instance image list zone=all project-id="$SCW_DEFAULT_PROJECT_ID" -otemplate="zone={{.Zone}} {{.ID}}" | xargs -L1 -P1 scw instance image delete with-snapshots=true
+# Clean volumes
+scw instance volume list zone=all project-id="$SCW_DEFAULT_PROJECT_ID" -otemplate="zone={{.Zone}} {{.ID}}" | xargs -L1 -P1 scw instance volume delete
+
+# A security group will be created alongside the server during packer execution.
+# We need to delete this security group before deleting the project
+scw instance security-group list zone=all project-id="$SCW_DEFAULT_PROJECT_ID" -otemplate="zone={{.Zone}} {{.ID}}" | xargs -L1 -P1 scw instance security-group delete
+
+scw account project delete project-id="$SCW_DEFAULT_PROJECT_ID"

e2e_tests/test.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+## This script will:
+# - create a temporary project, configure it in environment
+# - Run packer e2e tests
+# - Tries to delete project
+
+if [ "$PACKER_UPDATE_CASSETTES" == "true" ]
+then
+  PROJECT_ID=$(scw account project create -otemplate="{{.ID}}")
+  export SCW_DEFAULT_PROJECT_ID=$PROJECT_ID
+
+  echo Running tests with new project $SCW_DEFAULT_PROJECT_ID
+else
+  export SCW_ACCESS_KEY=SCWXXXXXXXXXXXXXFAKE
+  export SCW_SECRET_KEY=11111111-1111-1111-1111-111111111111
+  export SCW_DEFAULT_PROJECT_ID=11111111-1111-1111-1111-111111111111
+  echo Using cassettes, no test project was created
+fi
+
+go test ../internal/tests -v
+TEST_RESULT=$?
+
+if [ "$PACKER_UPDATE_CASSETTES" == "true" ]
+then
+  ./clean.sh $SCW_DEFAULT_PROJECT_ID
+fi
+
+exit $TEST_RESULT

e2e_tests/tmp.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+export SCW_DEFAULT_PROJECT_ID=c276a890-0f62-4d02-a5c5-9f86d615a029
+
+scw instance image list zone=all project-id="$SCW_DEFAULT_PROJECT_ID" -otemplate="zone={{.Zone}} {{.ID}}" | xargs -L1 -P1 scw instance image delete with-snapshots=true
+
+# A security group will be created alongside the server during packer execution.
+# We need to delete this security group before deleting the project
+scw instance security-group list zone=all project-id="$SCW_DEFAULT_PROJECT_ID" -otemplate="zone={{.Zone}} {{.ID}}" | xargs -L1 -P1 scw instance security-group delete
+
+scw account project delete project-id="$SCW_DEFAULT_PROJECT_ID"

go.mod

@@ -9,8 +9,10 @@ require (
 	github.com/hashicorp/packer-plugin-sdk v0.5.4
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30
+	github.com/stretchr/testify v1.8.4
 	github.com/zclconf/go-cty v1.13.3
 	golang.org/x/crypto v0.32.0
+	gopkg.in/dnaeon/go-vcr.v4 v4.0.2
 )
 
 require (
@@ -28,6 +30,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.114 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dylanmei/iso8601 v0.1.0 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
@@ -77,6 +80,7 @@ require (
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
 	github.com/packer-community/winrmcp v0.0.0-20180921211025-c76d91c1e7db // indirect
 	github.com/pkg/sftp v1.13.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
@@ -98,6 +102,7 @@ require (
 	google.golang.org/grpc v1.59.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 replace github.com/zclconf/go-cty => github.com/nywilken/go-cty v1.13.3 // added by packer-sdc fix as noted in github.com/hashicorp/packer-plugin-sdk/issues/187

go.sum

@@ -470,6 +470,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.2 h1:7T5VYf2ifyK01ETHbJPl5A6XTpUljD4Trw3GEDcdedk=
+gopkg.in/dnaeon/go-vcr.v4 v4.0.2/go.mod h1:65yxh9goQVrudqofKtHA4JNFWd6XZRkWfKN4YpMx7KI=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/checks/image.go

@@ -0,0 +1,77 @@
+package checks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/scaleway/packer-plugin-scaleway/internal/tester"
+	"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
+	"github.com/scaleway/scaleway-sdk-go/scw"
+)
+
+var _ tester.PackerCheck = (*ImageCheck)(nil)
+
+func Image(zone scw.Zone, name string) *ImageCheck {
+	return &ImageCheck{
+		zone:      zone,
+		imageName: name,
+	}
+}
+
+type ImageCheck struct {
+	zone      scw.Zone
+	imageName string
+
+	rootVolumeType *string
+	size           *scw.Size
+}
+
+func (c *ImageCheck) RootVolumeType(rootVolumeType string) *ImageCheck {
+	c.rootVolumeType = &rootVolumeType
+
+	return c
+}
+
+func (c *ImageCheck) SizeInGb(size uint64) *ImageCheck {
+	c.size = scw.SizePtr(scw.Size(size) * scw.GB)
+
+	return c
+}
+
+func (c *ImageCheck) Check(ctx context.Context) error {
+	testCtx := tester.ExtractCtx(ctx)
+	api := instance.NewAPI(testCtx.ScwClient)
+
+	resp, err := api.ListImages(&instance.ListImagesRequest{
+		Name:    &c.imageName,
+		Zone:    c.zone,
+		Project: &testCtx.ProjectID,
+	}, scw.WithAllPages(), scw.WithContext(ctx))
+	if err != nil {
+		return fmt.Errorf("failed to list images: %w", err)
+	}
+
+	if len(resp.Images) == 0 {
+		return fmt.Errorf("image %s not found, no images found", c.imageName)
+	}
+
+	if len(resp.Images) > 1 {
+		return fmt.Errorf("multiple images found with name %s", c.imageName)
+	}
+
+	image := resp.Images[0]
+
+	if image.Name != c.imageName {
+		return fmt.Errorf("image name %s does not match expected %s", image.Name, c.imageName)
+	}
+
+	if c.rootVolumeType != nil && string(image.RootVolume.VolumeType) != *c.rootVolumeType {
+		return fmt.Errorf("image root volume type %s does not match expected %s", image.RootVolume.VolumeType, *c.rootVolumeType)
+	}
+
+	if c.size != nil && image.RootVolume.Size != *c.size {
+		return fmt.Errorf("image size %d does not match expected %d", image.RootVolume.Size, *c.size)
+	}
+
+	return nil
+}

internal/checks/snapshot.go

@@ -0,0 +1,64 @@
+package checks
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/scaleway/packer-plugin-scaleway/internal/tester"
+	"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
+	"github.com/scaleway/scaleway-sdk-go/scw"
+)
+
+type SnapshotCheck struct {
+	zone         scw.Zone
+	snapshotName string
+
+	size *scw.Size
+}
+
+func (c *SnapshotCheck) SizeInGB(size uint64) *SnapshotCheck {
+	c.size = scw.SizePtr(scw.Size(size) * scw.GB)
+
+	return c
+}
+
+func (c *SnapshotCheck) Check(ctx context.Context) error {
+	testCtx := tester.ExtractCtx(ctx)
+	api := instance.NewAPI(testCtx.ScwClient)
+
+	resp, err := api.ListSnapshots(&instance.ListSnapshotsRequest{
+		Zone:    c.zone,
+		Name:    &c.snapshotName,
+		Project: &testCtx.ProjectID,
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(resp.Snapshots) == 0 {
+		return fmt.Errorf("snapshot %s not found, no snapshots found", c.snapshotName)
+	}
+
+	if len(resp.Snapshots) > 1 {
+		return fmt.Errorf("multiple snapshots found with name %s", c.snapshotName)
+	}
+
+	snapshot := resp.Snapshots[0]
+
+	if snapshot.Name != c.snapshotName {
+		return fmt.Errorf("snapshot name %s does not match expected snapshot name %s", snapshot.Name, c.snapshotName)
+	}
+
+	if c.size != nil && snapshot.Size != *c.size {
+		return fmt.Errorf("snapshot size %d does not match expected size %d", snapshot.Size, *c.size)
+	}
+
+	return nil
+}
+
+func Snapshot(zone scw.Zone, snapshotName string) *SnapshotCheck {
+	return &SnapshotCheck{
+		zone:         zone,
+		snapshotName: snapshotName,
+	}
+}