ci(e2e): add backwards compatibility test (#1762)

Adds a `backwards.toml` that ensures backwards compatibility of `main`
with `v0.4.0` our first omega LTS version.

issue: #1537

Makefile

@@ -83,7 +83,7 @@ devnet-clean: ## Deletes devnet1 containers
 .PHONY: e2e-ci
 e2e-ci: ## Runs all e2e CI tests
 	@go install github.com/omni-network/omni/e2e
-	@cd e2e && ./run-multiple.sh manifests/devnet1.toml manifests/fuzzyhead.toml manifests/ci.toml
+	@cd e2e && ./run-multiple.sh manifests/devnet1.toml manifests/fuzzyhead.toml manifests/ci.toml manifests/backwards.toml
 
 .PHONY: e2e-run
 e2e-run: ## Run specific e2e manifest (MANIFEST=single, MANIFEST=devnet1, etc). Note container remain running after the test.

e2e/app/definition.go

@@ -196,7 +196,7 @@ func newBackends(ctx context.Context, cfg DefinitionConfig, testnet types.Testne
 func adaptCometTestnet(ctx context.Context, manifest types.Manifest, testnet *e2e.Testnet, imgTag string) (*e2e.Testnet, error) {
 	testnet.Dir = runsDir(testnet.File)
 	testnet.VoteExtensionsEnableHeight = 1
-	testnet.UpgradeVersion = "omniops/halo:" + imgTag
+	testnet.UpgradeVersion = "omniops/halovisor:" + imgTag // Currently only support upgrading to "latest" version
 
 	for i := range testnet.Nodes {
 		var err error
@@ -225,7 +225,12 @@ func adaptNode(ctx context.Context, manifest types.Manifest, node *e2e.Node, tag
 		tag = manifest.PinnedHaloTag
 	}
 
-	node.Version = "omniops/halovisor:" + tag
+	// Override default comet version with our own, see github.com/cometbft/cometbft@v0.38.11/test/e2e/pkg/testnet.go:36
+	const cometLocalVersion = "cometbft/e2e-node:local-version"
+	if node.Version == cometLocalVersion {
+		node.Version = "omniops/halovisor:" + tag
+	}
+
 	node.PrivvalKey = valKey.PrivKey
 	node.NodeKey = nodeKey.PrivKey
 

e2e/app/perturb.go

@@ -127,7 +127,18 @@ func perturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbat
 		}
 
 	case e2e.PerturbationUpgrade:
-		return nil, errors.New("upgrade perturbation not supported")
+		log.Info(ctx, "Perturb node: upgrade", "from", node.Version, "to", testnet.UpgradeVersion)
+		if err := docker.ExecCompose(ctx, testnet.Dir, "stop", name); err != nil {
+			return nil, errors.Wrap(err, "stop service")
+		}
+
+		if err := docker.ReplaceUpgradeImage(testnet.Dir, name); err != nil {
+			return nil, errors.Wrap(err, "upgrade service")
+		}
+
+		if err := docker.ExecCompose(ctx, testnet.Dir, "start", name); err != nil {
+			return nil, errors.Wrap(err, "start service")
+		}
 
 	default:
 		return nil, errors.New("unexpected perturbation type", "type", perturbation)

e2e/docker/compose.yaml.tmpl

@@ -17,7 +17,7 @@ services:
     labels:
       e2e: true
     container_name: {{ .Name }}
-    image: {{ .Version }}
+    image: {{ .Version }}{{if ne .Version $.UpgradeVersion}} # Upgrade {{ .Name }}:{{ $.UpgradeVersion }}{{end}}
     restart: unless-stopped
     init: true
     ports:

e2e/docker/docker.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	osexec "os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"sync"
 	"text/template"
@@ -77,16 +78,17 @@ func NewProvider(testnet types.Testnet, infd types.InfrastructureData, imgTag st
 // any of these operations fail.
 func (p *Provider) Setup() error {
 	def := ComposeDef{
-		Network:     true,
-		NetworkName: p.testnet.Name,
-		NetworkCIDR: p.testnet.IP.String(),
-		BindAll:     false,
-		Nodes:       p.testnet.Nodes,
-		OmniEVMs:    p.testnet.OmniEVMs,
-		Anvils:      p.testnet.AnvilChains,
-		Relayer:     true,
-		Prometheus:  p.testnet.Prometheus,
-		Monitor:     true,
+		Network:        true,
+		NetworkName:    p.testnet.Name,
+		NetworkCIDR:    p.testnet.IP.String(),
+		BindAll:        false,
+		UpgradeVersion: p.testnet.UpgradeVersion,
+		Nodes:          p.testnet.Nodes,
+		OmniEVMs:       p.testnet.OmniEVMs,
+		Anvils:         p.testnet.AnvilChains,
+		Relayer:        true,
+		Prometheus:     p.testnet.Prometheus,
+		Monitor:        true,
 	}
 	def = SetImageTags(def, p.testnet.Manifest, p.omniTag)
 
@@ -166,10 +168,11 @@ func (p *Provider) StartNodes(ctx context.Context, nodes ...*e2e.Node) error {
 }
 
 type ComposeDef struct {
-	Network     bool
-	NetworkName string
-	NetworkCIDR string
-	BindAll     bool
+	Network        bool
+	NetworkName    string
+	NetworkCIDR    string
+	BindAll        bool
+	UpgradeVersion string
 
 	Nodes    []*e2e.Node
 	OmniEVMs []types.OmniEVM
@@ -346,3 +349,34 @@ func Exec(ctx context.Context, args ...string) error {
 
 	return nil
 }
+
+// ReplaceUpgradeImage replaces the docker image of the provided service with the
+// version specified in comments. Expected format below upgrades node0 from main v1.0 to v2.0:
+//
+//	  services:
+//		 node0:
+//		   labels:
+//		     e2e: true
+//		   container_name: node0
+//		   image: omniops/halo:main # Upgrade node0:omniops/halo:v1.0
+//		   restart: unless-stopped
+func ReplaceUpgradeImage(dir, service string) error {
+	before, err := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
+	if err != nil {
+		return errors.Wrap(err, "read compose file")
+	}
+
+	regex := regexp.MustCompile(`(\s+image: ).+\s#\sUpgrade ` + service + `:(.*)`)
+
+	after := regex.ReplaceAll(before, []byte("$1$2"))
+	if bytes.Equal(before, after) {
+		return errors.New("no upgrade image found")
+	}
+
+	err = os.WriteFile(filepath.Join(dir, "docker-compose.yml"), after, 0o644)
+	if err != nil {
+		return errors.Wrap(err, "write compose file")
+	}
+
+	return nil
+}

e2e/docker/docker_test.go

@@ -30,6 +30,7 @@ func TestComposeTemplate(t *testing.T) {
 		name       string
 		tag        string
 		isEmpheral bool
+		upgrade    string
 	}{
 		{
 			name:       "commit",
@@ -40,6 +41,7 @@ func TestComposeTemplate(t *testing.T) {
 			name:       "empheral_network",
 			tag:        "main",
 			isEmpheral: true,
+			upgrade:    "omniops/halo:v1.0",
 		},
 	}
 
@@ -53,6 +55,7 @@ func TestComposeTemplate(t *testing.T) {
 			require.NoError(t, err)
 			en := enode.NewV4(&key.PublicKey, ipNet.IP, 30303, 30303)
 
+			const node0 = "node0"
 			dir := t.TempDir()
 			testnet := types.Testnet{
 				Manifest: types.Manifest{
@@ -65,11 +68,12 @@ func TestComposeTemplate(t *testing.T) {
 					Dir:        dir,
 					Prometheus: true,
 					Nodes: []*e2e.Node{{
-						Name:       "node0",
+						Name:       node0,
 						Version:    "omniops/halo:" + test.tag,
 						InternalIP: ipNet.IP,
 						ProxyPort:  8584,
 					}},
+					UpgradeVersion: "omniops/halo:" + test.tag,
 				},
 				OmniEVMs: []types.OmniEVM{
 					{
@@ -110,6 +114,10 @@ func TestComposeTemplate(t *testing.T) {
 				testnet.Network = netconf.Devnet
 			}
 
+			if test.upgrade != "" {
+				testnet.UpgradeVersion = test.upgrade
+			}
+
 			p := docker.NewProvider(testnet, types.InfrastructureData{}, test.tag)
 			require.NoError(t, err)
 
@@ -119,6 +127,21 @@ func TestComposeTemplate(t *testing.T) {
 			require.NoError(t, err)
 
 			tutil.RequireGoldenBytes(t, bz)
+
+			t.Run("upgrade", func(t *testing.T) {
+				t.Parallel()
+				err := docker.ReplaceUpgradeImage(dir, node0)
+				if test.upgrade == "" {
+					require.Error(t, err)
+					return
+				}
+				require.NoError(t, err)
+
+				bz, err := os.ReadFile(filepath.Join(dir, "docker-compose.yml"))
+				require.NoError(t, err)
+
+				tutil.RequireGoldenBytes(t, bz)
+			})
 		})
 	}
 }

e2e/docker/testdata/TestComposeTemplate_empheral_network.golden

@@ -14,7 +14,7 @@ services:
     labels:
       e2e: true
     container_name: node0
-    image: omniops/halo:main
+    image: omniops/halo:main # Upgrade node0:omniops/halo:v1.0
     restart: unless-stopped
     init: true
     ports:

e2e/docker/testdata/TestComposeTemplate_empheral_network_upgrade.golden

@@ -0,0 +1,146 @@
+version: '2.4'
+networks:
+  test:
+    labels:
+      e2e: true
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+      - subnet: 10.186.73.0/24
+
+services:
+  node0:
+    labels:
+      e2e: true
+    container_name: node0
+    image: omniops/halo:v1.0
+    restart: unless-stopped
+    init: true
+    ports:
+    - 26656 # Consensus P2P
+    - 8584:26657 # Consensus RPC
+    - 6060 # Pprof
+    volumes:
+    - ./node0/config:/halo/config
+    - ./node0/data:/halo/data
+    networks:
+      test:
+        ipv4_address: 10.186.73.0
+
+  mock_rollup:
+    labels:
+      e2e: true
+    container_name: mock_rollup
+    platform: linux/amd64
+    image: omniops/anvilproxy:main
+    environment:
+      - ANVILPROXY_CHAIN_ID=99
+      - ANVILPROXY_BLOCK_TIME=1
+      - ANVILPROXY_SLOTS_IN_AN_EPOCH=4 # Finality in 4*2*BlockPeriod
+
+    ports:
+      - 9000:8545
+    networks:
+      test:
+        ipv4_address: 10.186.73.0
+
+  mock_l1:
+    labels:
+      e2e: true
+    container_name: mock_l1
+    platform: linux/amd64
+    image: omniops/anvilproxy:main
+    environment:
+      - ANVILPROXY_CHAIN_ID=1
+      - ANVILPROXY_BLOCK_TIME=3600
+      - ANVILPROXY_SLOTS_IN_AN_EPOCH=4 # Finality in 4*2*BlockPeriod
+      - FORKPROXY_LOAD_STATE=/anvil/state.json
+    ports:
+      - 9000:8545
+    networks:
+      test:
+        ipv4_address: 10.186.73.0
+
+    volumes:
+      - path/to/anvil/state.json:/anvil/state.json
+
+
+  # Use geth as the omni EVMs.
+  omni_evm_0:
+    labels:
+      e2e: true
+    container_name: omni_evm_0
+    image: "ethereum/client-go:v1.14.7"
+    restart: unless-stopped
+    command:
+      - --config=/geth/config.toml
+      # Flags not available via config.toml
+      - --nat=extip:10.186.73.0
+      - --pprof
+      - --pprof.addr=0.0.0.0
+      - --metrics
+      - --graphql
+
+    ports:
+      - 8551 # Auth RPC
+      - 8000:8545 # HTTP RPC
+      - 30303 # Execution P2P
+      - 8546 # Websockets RPC
+      - 6060 # Prometheus metrics and pprof
+    healthcheck:
+      test: "nc -z localhost 8545"
+      interval: 1s
+      retries: 30
+    volumes:
+      - ./omni_evm_0:/geth
+    networks:
+      test:
+        ipv4_address: 10.186.73.0
+
+  relayer:
+    labels:
+      e2e: true
+    container_name: relayer
+    image: omniops/relayer:v2
+    restart: unless-stopped
+    ports:
+      - 26660 # Prometheus and pprof
+    volumes:
+      - ./relayer:/relayer
+    networks:
+      test:
+        ipv4_address: 10.186.73.200
+
+  monitor:
+    labels:
+      e2e: true
+    container_name: monitor
+    image: omniops/monitor:v3
+    restart: unless-stopped
+    ports:
+      - 26660 # Prometheus and pprof
+    volumes:
+      - ./monitor:/monitor
+    networks:
+      test:
+        ipv4_address: 10.186.73.201
+
+  prometheus:
+    labels:
+      e2e: true
+    container_name: prometheus
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.console.libraries=/usr/share/prometheus/console_libraries
+      - --web.console.templates=/usr/share/prometheus/consoles
+      - --enable-feature=exemplar-storage
+      - --enable-feature=agent
+    restart: unless-stopped
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    networks:
+      test:
+        ipv4_address: 10.186.73.202
+

e2e/manifests/backwards.toml

@@ -0,0 +1,17 @@
+# Backwards ensures backwards-compatibility with v0.4.0.
+network = "devnet"
+anvil_chains = ["mock_l1", "mock_l2"]
+
+multi_omni_evms = true
+
+[node.validator01]
+[node.validator02]
+mode = "archive"
+
+[node.validator03]
+version="omniops/halovisor:v0.4.0"
+perturb = ["upgrade"]
+
+[node.validator04]
+version="omniops/halovisor:v0.4.0"
+perturb = ["upgrade"]