chore: Switch the default operand image tag from latest to next and fix CI nightly job [RHIDP-3157] (#401)

* Switch the default operand image from 'quay.io/janus-idp/backstage-showcase:latest' to 'quay.io/rhdh/rhdh-hub-rhel9:latest'

'quay.io/rhdh/rhdh-hub-rhel9' is the downstream image of the Showcase image

* Fix the E2E tests using the latest stable image of the operand

User authentication is now needed to query the '/api/dynamic-plugins-info/loaded-plugins' API endpoint

* wip: show controller logs in case CR status is not updated

* Fix the upgrade path E2E tests by ensuring the CR on 1.1 is still using v1alpha1 as version

Otherwise, the 1.x operator is not aware of the v1alpha2 version

* Also display operand logs in case there is an assertion failure

* Adjust timeouts because the operand image takes a bit longer to start

* Bump setup-minikube action to the latest

It should hopefully fix the random network connection issues

* Separate upgrade and non-upgrade tests as different jobs

This will reduce the likeliness of random connectivity failures

* Revert "Switch the default operand image from 'quay.io/janus-idp/backstage-showcase:latest' to 'quay.io/rhdh/rhdh-hub-rhel9:latest'"

This reverts commit 8b81157.

* Switch the default operand image tag from 'latest' to 'next'

'next' is the development tag of the upstream Showcase image,
and should hopefully be stable.
The goal with this is to monitor how our tests behave against this 'next' tag. If it proves too unstable,
we might want to switch back to using the downstream RHDH image
(`quay.io/rhdh/rhdh-hub-rhel9:latest`).

Author: rm3l

.github/workflows/nightly.yaml

@@ -13,9 +13,13 @@ jobs:
       fail-fast: false
       matrix:
         branch: [ main, 1.2.x, 1.1.x ]
-    name: E2E Tests - ${{ matrix.branch }}
+        test_upgrade: [ 'true', 'false' ]
+        exclude:
+          - branch: 1.1.x # Testing upgrade from 1.1.x
+            test_upgrade: 'true'
+    name: 'E2E Tests - ${{ matrix.branch }} - upgrade=${{ matrix.test_upgrade }}'
     concurrency:
-      group: ${{ github.workflow }}-${{ matrix.branch }}
+      group: '${{ github.workflow }}-${{ matrix.branch }}-${{ matrix.test_upgrade }}'
       cancel-in-progress: true
     env:
       CONTAINER_ENGINE: podman
@@ -49,20 +53,17 @@ jobs:
 
       - name: Start Minikube
         if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
-        uses: medyagh/setup-minikube@317d92317e473a10540357f1f4b2878b80ee7b95 # v0.0.16
-        with:
-          addons: ingress
+        uses: medyagh/setup-minikube@d8c0eb871f6f455542491d86a574477bd3894533 # v0.0.18
 
       - name: Run E2E tests (Operator Upgrade path)
-        # Testing upgrade from 1.1.x
-        if: ${{ matrix.branch != '1.1.x' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
+        if: ${{ matrix.test_upgrade == 'true' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
         env:
           BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube
           IMG: ${{ env.OPERATOR_IMAGE }}
         run: make test-e2e-upgrade
 
       - name: Run E2E tests
-        if: ${{ steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
+        if: ${{ matrix.test_upgrade == 'false' && steps.operator-image-existence-checker.outputs.OPERATOR_IMAGE_EXISTS == 'true' }}
         env:
           BACKSTAGE_OPERATOR_TESTS_PLATFORM: minikube
           IMG: ${{ env.OPERATOR_IMAGE }}

bundle/manifests/backstage-default-config_v1_configmap.yaml

@@ -191,7 +191,8 @@ data:
               command:
                 - ./install-dynamic-plugins.sh
                 - /dynamic-plugins-root
-              image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next
+              # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               securityContext:
                 runAsNonRoot: true
@@ -218,7 +219,7 @@ data:
           containers:
             - name: backstage-backend
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               args:
                 - "--config"

bundle/manifests/backstage-operator.clusterserviceversion.yaml

@@ -21,7 +21,7 @@ metadata:
         }
       ]
     capabilities: Seamless Upgrades
-    createdAt: "2024-07-08T12:45:16Z"
+    createdAt: "2024-07-16T16:00:50Z"
     operatorframework.io/suggested-namespace: backstage-system
     operators.operatorframework.io/builder: operator-sdk-v1.33.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3
@@ -213,7 +213,7 @@ spec:
                 - name: RELATED_IMAGE_postgresql
                   value: quay.io/fedora/postgresql-15:latest
                 - name: RELATED_IMAGE_backstage
-                  value: quay.io/janus-idp/backstage-showcase:latest
+                  value: quay.io/janus-idp/backstage-showcase:next
                 image: quay.io/janus-idp/operator:0.3.0
                 livenessProbe:
                   httpGet:
@@ -317,6 +317,6 @@ spec:
   relatedImages:
   - image: quay.io/fedora/postgresql-15:latest
     name: postgresql
-  - image: quay.io/janus-idp/backstage-showcase:latest
+  - image: quay.io/janus-idp/backstage-showcase:next
     name: backstage
   version: 0.3.0

config/manager/default-config/deployment.yaml

@@ -38,7 +38,8 @@ spec:
           command:
             - ./install-dynamic-plugins.sh
             - /dynamic-plugins-root
-          image: quay.io/janus-idp/backstage-showcase:latest # will be replaced with the actual image quay.io/janus-idp/backstage-showcase:next
+          # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
+          image: quay.io/janus-idp/backstage-showcase:next
           imagePullPolicy: IfNotPresent
           securityContext:
             runAsNonRoot: true
@@ -65,7 +66,7 @@ spec:
       containers:
         - name: backstage-backend
           # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-          image: quay.io/janus-idp/backstage-showcase:latest
+          image: quay.io/janus-idp/backstage-showcase:next
           imagePullPolicy: IfNotPresent
           args:
             - "--config"

config/manager/manager.yaml

@@ -76,7 +76,7 @@ spec:
         - name: RELATED_IMAGE_postgresql
           value: quay.io/fedora/postgresql-15:latest
         - name: RELATED_IMAGE_backstage
-          value: quay.io/janus-idp/backstage-showcase:latest
+          value: quay.io/janus-idp/backstage-showcase:next
         image: controller:latest
         name: manager
         securityContext:

examples/bs1.yaml

@@ -2,6 +2,3 @@ apiVersion: rhdh.redhat.com/v1alpha2
 kind: Backstage
 metadata:
   name: bs1
-
-
-

examples/rhdh-cr-with-app-configs.yaml

@@ -47,10 +47,16 @@ data:
     backend:
       auth:
         externalAccess:
-            - type: legacy
-              options:
-                subject: legacy-default-config
-                secret: "${BACKEND_SECRET}"
+          - type: legacy
+            options:
+              subject: legacy-default-config
+              secret: "${BACKEND_SECRET}"
+    auth:
+      environment: development
+      providers:
+        guest:
+          # using the guest user to query the '/api/dynamic-plugins-info/loaded-plugins' endpoint.
+          dangerouslyAllowOutsideDevelopment: true
 ---
 apiVersion: v1
 kind: Secret
@@ -128,7 +134,7 @@ data:
                     initialDelay: { seconds: 15}
       - package: ./dynamic-plugins/dist/backstage-plugin-techdocs-backend-dynamic
         pluginConfig:
-          # Reference documentation http://backstage.io/docs/features/techdocs/configuration
+          # Reference documentation https://backstage.io/docs/features/techdocs/configuration
           # Note: After experimenting with basic setup, use CI/CD to generate docs
           # and an external cloud storage when deploying TechDocs for production use-case.
           # https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-basic-to-recommended-deployment-approach

tests/e2e/e2e_suite_test.go

@@ -223,6 +223,15 @@ func verifyControllerUp(g Gomega, managerPodLabel string) {
 	g.Expect(string(status)).Should(Equal("Running"), fmt.Sprintf("controller pod in %s status", status))
 }
 
+func getPodLogs(ns string, label string) string {
+	cmd := exec.Command(helper.GetPlatformTool(), "logs",
+		"-l", label,
+		"-n", ns,
+	)
+	output, _ := helper.Run(cmd)
+	return string(output)
+}
+
 func uninstallOperator() {
 	switch testMode {
 	case rhdhLatestTestMode, rhdhNextTestMode, rhdhAirgapTestMode:

tests/e2e/e2e_test.go

@@ -15,7 +15,11 @@
 package e2e
 
 import (
+	"crypto/tls"
+	"encoding/json"
 	"fmt"
+	"io"
+	"net/http"
 	"os/exec"
 	"path/filepath"
 	"strconv"
@@ -78,9 +82,44 @@ var _ = Describe("Backstage Operator E2E", func() {
 				crName:     "bs-app-config",
 				additionalApiEndpointTests: []helper.ApiEndpointTest{
 					{
-						Endpoint:               "/api/dynamic-plugins-info/loaded-plugins",
+						Endpoint: "/api/dynamic-plugins-info/loaded-plugins",
+						BearerTokenRetrievalFn: func(baseUrl string) (string, error) { // Authenticated endpoint that does not accept service tokens
+							url := fmt.Sprintf("%s/api/auth/guest/refresh", baseUrl)
+							tr := &http.Transport{
+								TLSClientConfig: &tls.Config{
+									InsecureSkipVerify: true, // #nosec G402 -- test code only, not used in production
+								},
+							}
+							httpClient := &http.Client{Transport: tr}
+							req, err := http.NewRequest("GET", url, nil)
+							if err != nil {
+								return "", fmt.Errorf("error while building request to GET %q: %w", url, err)
+							}
+							req.Header.Add("Accept", "application/json")
+							resp, err := httpClient.Do(req)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to GET %q: %w", url, err)
+							}
+							defer resp.Body.Close()
+							body, err := io.ReadAll(resp.Body)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to read response body from 'GET %q': %w", url, err)
+							}
+							if resp.StatusCode != 200 {
+								return "", fmt.Errorf("expected status code 200, but got %d in response to 'GET %q', body: %s", resp.StatusCode, url, string(body))
+							}
+							var authResponse helper.BackstageAuthRefreshResponse
+							err = json.Unmarshal(body, &authResponse)
+							if err != nil {
+								return "", fmt.Errorf("error while trying to decode response body from 'GET %q': %w", url, err)
+							}
+							return authResponse.BackstageIdentity.Token, nil
+						},
 						ExpectedHttpStatusCode: 200,
 						BodyMatcher: SatisfyAll(
+							ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-quay-dynamic"),
+							ContainSubstring("@janus-idp/backstage-scaffolder-backend-module-regex-dynamic"),
+							ContainSubstring("roadiehq-scaffolder-backend-module-utils-dynamic"),
 							ContainSubstring("backstage-plugin-catalog-backend-module-github-dynamic"),
 							ContainSubstring("backstage-plugin-techdocs-backend-dynamic"),
 							ContainSubstring("backstage-plugin-catalog-backend-module-gitlab-dynamic")),
@@ -187,7 +226,7 @@ var _ = Describe("Backstage Operator E2E", func() {
 })
 
 func ensureRouteIsReachable(ns string, crName string, additionalApiEndpointTests []helper.ApiEndpointTest) {
-	Eventually(helper.VerifyBackstageRoute, time.Minute, time.Second).
+	Eventually(helper.VerifyBackstageRoute, 5*time.Minute, time.Second).
 		WithArguments(ns, crName, additionalApiEndpointTests).
 		Should(Succeed())
 }

tests/e2e/e2e_upgrade_test.go

@@ -16,6 +16,7 @@ package e2e
 
 import (
 	"fmt"
+	"io"
 	"os/exec"
 	"path/filepath"
 	"time"
@@ -49,13 +50,10 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 	When("Previous version of operator is installed and CR is created", func() {
 
 		const managerPodLabel = "control-plane=controller-manager"
+		const crName = "my-backstage-app"
 
 		// 0.1.3 is the version of the operator in the 1.1.x branch
 		var fromDeploymentManifest = filepath.Join(projectDir, "tests", "e2e", "testdata", "backstage-operator-0.1.3.yaml")
-		var (
-			crName = "bs1"
-			crPath = filepath.Join(projectDir, "examples", "bs1.yaml")
-		)
 
 		BeforeEach(func() {
 			if testMode != defaultDeployTestMode {
@@ -71,9 +69,22 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 			Expect(err).ShouldNot(HaveOccurred())
 			EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed())
 
-			cmd = exec.Command(helper.GetPlatformTool(), "apply", "-f", crPath, "-n", ns)
+			cmd = exec.Command(helper.GetPlatformTool(), "-n", ns, "create", "-f", "-")
+			stdin, err := cmd.StdinPipe()
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+			go func() {
+				defer stdin.Close()
+				_, _ = io.WriteString(stdin, fmt.Sprintf(`
+apiVersion: rhdh.redhat.com/v1alpha1
+kind: Backstage
+metadata:
+  name: my-backstage-app
+  namespace: %s
+`, ns))
+			}()
 			_, err = helper.Run(cmd)
 			Expect(err).ShouldNot(HaveOccurred())
+
 			// Reason is DeployOK in 1.1.x, but was renamed to Deployed in 1.2
 			Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"DeployOK"`).Should(Succeed())
 		})
@@ -89,17 +100,21 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 		It("should successfully reconcile existing CR when upgrading the operator", func() {
 			By("Upgrading the operator", func() {
 				installOperatorWithMakeDeploy(false)
-				EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, time.Second).WithArguments(managerPodLabel).Should(Succeed())
+				EventuallyWithOffset(1, verifyControllerUp, 5*time.Minute, 3*time.Second).WithArguments(managerPodLabel).Should(Succeed())
 			})
 
 			By("checking the status of the existing CR")
-			Eventually(helper.VerifyBackstageCRStatus, time.Minute, time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).Should(Succeed())
+			Eventually(helper.VerifyBackstageCRStatus, 5*time.Minute, 3*time.Second).WithArguments(ns, crName, `"reason":"Deployed"`).
+				Should(Succeed(), func() string {
+					return fmt.Sprintf("=== Operator logs ===\n%s\n", getPodLogs(_namespace, managerPodLabel))
+				})
 
 			By("checking the Backstage operand pod")
+			crLabel := fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName)
 			Eventually(func(g Gomega) {
 				// Get pod name
 				cmd := exec.Command(helper.GetPlatformTool(), "get",
-					"pods", "-l", fmt.Sprintf("rhdh.redhat.com/app=backstage-%s", crName),
+					"pods", "-l", crLabel,
 					"-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .metadata.name }}"+
 						"{{ \"\\n\" }}{{ end }}{{ end }}",
 					"-n", ns,
@@ -108,7 +123,9 @@ var _ = Describe("Operator upgrade with existing instances", func() {
 				g.Expect(err).ShouldNot(HaveOccurred())
 				podNames := helper.GetNonEmptyLines(string(podOutput))
 				g.Expect(podNames).Should(HaveLen(1), fmt.Sprintf("expected 1 Backstage operand pod(s) running, but got %d", len(podNames)))
-			}, 5*time.Minute, time.Second).Should(Succeed())
+			}, 10*time.Minute, 5*time.Second).Should(Succeed(), func() string {
+				return fmt.Sprintf("=== Operand logs ===\n%s\n", getPodLogs(ns, crLabel))
+			})
 		})
 	})
 

tests/e2e/testdata/backstage-operator-0.1.3.yaml

@@ -739,7 +739,7 @@ data:
                 - name: NPM_CONFIG_USERCONFIG
                   value: /opt/app-root/src/.npmrc.dynamic-plugins
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               name: install-dynamic-plugins
               volumeMounts:
@@ -761,7 +761,7 @@ data:
           containers:
             - name: backstage-backend
               # image will be replaced by the value of the `RELATED_IMAGE_backstage` env var, if set
-              image: quay.io/janus-idp/backstage-showcase:latest
+              image: quay.io/janus-idp/backstage-showcase:next
               imagePullPolicy: IfNotPresent
               args:
                 - "--config"
@@ -949,7 +949,7 @@ spec:
             - name: RELATED_IMAGE_postgresql
               value: quay.io/fedora/postgresql-15:latest
             - name: RELATED_IMAGE_backstage
-              value: quay.io/janus-idp/backstage-showcase:latest
+              value: quay.io/janus-idp/backstage-showcase:next
           # TODO(asoro): Default image is 'quay.io/janus-idp/operator:0.1.3' on 1.1.x,
           # but replaced by the one from RHDH, because the Janus-IDP image expires after 14d if not updated.
           image: quay.io/rhdh/rhdh-rhel9-operator:1.1