From d4f94ee4b340232b556664873dbefb39be3b2d5d Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Tue, 29 Aug 2023 19:27:13 +0530
Subject: [PATCH 01/11] set validator restartPolicy to never, run smoke tests
 for PR, update resources for multi-validator smoke test

---
 .github/workflows/smoke-tests.yaml                   | 4 ++++
 charts/devnet/templates/chains/cosmos/validator.yaml | 1 +
 tests/smoke/ci/multi-validator.yaml                  | 8 ++++++--
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml
index 9f30f3bbf..ecb6c1d9b 100644
--- a/.github/workflows/smoke-tests.yaml
+++ b/.github/workflows/smoke-tests.yaml
@@ -1,6 +1,10 @@
 name: Smoke tests
 
 on:
+  # debug CI for PR
+  pull_request:
+    branches:
+      - main
   # Run every 6 hours
   schedule:
     - cron: '0 */6 * * *'
diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index 5461cc68e..6be1c2f4f 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -44,6 +44,7 @@ spec:
         app.kubernetes.io/name: {{ $chain.name }}-validator
         app.kubernetes.io/version: {{ $.Chart.AppVersion }}
     spec:
+      restartPolicy: Never
       {{- include "imagePullSecrets" $chain | indent 6 }}
       initContainers:
         {{- if $toBuild }}
diff --git a/tests/smoke/ci/multi-validator.yaml b/tests/smoke/ci/multi-validator.yaml
index 3fefd8a80..64f63c31c 100644
--- a/tests/smoke/ci/multi-validator.yaml
+++ b/tests/smoke/ci/multi-validator.yaml
@@ -7,8 +7,12 @@ chains:
       rpc: 26653
       exposer: 38083
     resources:
-      cpu: "0.3"
-      memory: 600M
+      limits:
+        cpu: "0.4"
+        memory: 1Gi
+      requests:
+        cpu: "0.2"
+        memory: 600M
     faucet:
       concurrency: 2
       resources:

From 2032491431c20ef497f1c9305e3f98e28e5350ae Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Tue, 29 Aug 2023 19:35:21 +0530
Subject: [PATCH 02/11] set restartPolicy to always, only supported

---
 charts/devnet/templates/chains/cosmos/validator.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index 6be1c2f4f..0ee43a1f6 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -44,7 +44,7 @@ spec:
         app.kubernetes.io/name: {{ $chain.name }}-validator
         app.kubernetes.io/version: {{ $.Chart.AppVersion }}
     spec:
-      restartPolicy: Never
+      restartPolicy: Always
       {{- include "imagePullSecrets" $chain | indent 6 }}
       initContainers:
         {{- if $toBuild }}

From a5f6f3ce08c9a0f0a258ab0dd1c609bc5d25f056 Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 08:09:27 +0530
Subject: [PATCH 03/11] manually run validator 0

---
 charts/devnet/templates/chains/cosmos/validator.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index 0ee43a1f6..543e58bf2 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -107,6 +107,8 @@ spec:
               cp $CHAIN_DIR/cosmovisor/genesis/bin/$CHAIN_BIN /usr/bin
               {{- end }}
 
+              exit 0
+
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
                 echo "Genesis file exists, exiting early"
                 exit 0
@@ -166,6 +168,7 @@ spec:
             - bash
             - "-c"
             - |
+              sleep 1000000000
               # Starting the chain
               {{- if $toBuild }}
               cp $CHAIN_DIR/cosmovisor/genesis/bin/$CHAIN_BIN /usr/bin
@@ -181,6 +184,7 @@ spec:
                   - "-c"
                   - "-e"
                   - |
+                    exit 0
                     VAL_INDEX=${HOSTNAME##*-}
                     VAL_NAME=$(jq -r ".validators[$VAL_INDEX].name" /configs/keys.json)
                     [[ $VAL_NAME == "null" ]] && VAL_NAME="validator-$VAL_INDEX"

From 11eee64f1dcae7a7eab038b30732712861664568 Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 08:22:01 +0530
Subject: [PATCH 04/11] remove last printing of the whole config.toml file

---
 charts/devnet/templates/chains/cosmos/validator.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index 543e58bf2..23cbf8926 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -107,8 +107,6 @@ spec:
               cp $CHAIN_DIR/cosmovisor/genesis/bin/$CHAIN_BIN /usr/bin
               {{- end }}
 
-              exit 0
-
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
                 echo "Genesis file exists, exiting early"
                 exit 0
@@ -145,9 +143,6 @@ spec:
               GENESIS_NODE_P2P=$NODE_ID@$GENESIS_HOST.$NAMESPACE.svc.cluster.local:26656
               echo "Node P2P: $GENESIS_NODE_P2P"
               sed -i "s/persistent_peers = \"\"/persistent_peers = \"$GENESIS_NODE_P2P\"/g" $CHAIN_DIR/config/config.toml
-
-              echo "Printing the whole config.toml file"
-              cat $CHAIN_DIR/config/config.toml
           resources: {{- include "devnet.node.resources" ( dict "node" $chain "context" $ ) | trim | nindent 12 }}
           volumeMounts:
             - mountPath: {{ $chain.home }}
@@ -168,7 +163,6 @@ spec:
             - bash
             - "-c"
             - |
-              sleep 1000000000
               # Starting the chain
               {{- if $toBuild }}
               cp $CHAIN_DIR/cosmovisor/genesis/bin/$CHAIN_BIN /usr/bin
@@ -184,7 +178,6 @@ spec:
                   - "-c"
                   - "-e"
                   - |
-                    exit 0
                     VAL_INDEX=${HOSTNAME##*-}
                     VAL_NAME=$(jq -r ".validators[$VAL_INDEX].name" /configs/keys.json)
                     [[ $VAL_NAME == "null" ]] && VAL_NAME="validator-$VAL_INDEX"

From 9266fcf2b21eb4967135e52b3aa13fc2a98dc7fd Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 13:55:09 +0530
Subject: [PATCH 05/11] try checking a file against genesis-node genesis file

---
 .../templates/chains/cosmos/validator.yaml      | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index 23cbf8926..4eeadc1ec 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -107,9 +107,18 @@ spec:
               cp $CHAIN_DIR/cosmovisor/genesis/bin/$CHAIN_BIN /usr/bin
               {{- end }}
 
+              curl http://$GENESIS_HOST.$NAMESPACE.svc.cluster.local:$GENESIS_PORT/genesis -o /tmp/genesis.json
+              echo "Genesis file that we got....."
+
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
-                echo "Genesis file exists, exiting early"
-                exit 0
+                diff "$file1" "$file2" > /dev/null
+                if [ $? -eq 0 ]; then
+                  echo "Genesis file exists, exiting early"
+                  exit 0
+                else
+                  echo "Genesis found does not match..., performing cleanup"
+                  rm -rf $CHAIN_DIR
+                fi
               fi
 
               RECOVER=true
@@ -127,8 +136,8 @@ spec:
                 $CHAIN_BIN keys add $VAL_NAME --keyring-backend="test"
               fi
 
-              curl http://$GENESIS_HOST.$NAMESPACE.svc.cluster.local:$GENESIS_PORT/genesis -o $CHAIN_DIR/config/genesis.json
-              echo "Genesis file that we got....."
+              echo "Copy genesis.json to correct place"
+              cp /tmp/genesis.json $CHAIN_DIR/config/genesis.json
               cat $CHAIN_DIR/config/genesis.json
 
               echo "Setup config files"

From 44fadc465b368a21ad1a9b8d00dbd91a30c803aa Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 16:32:19 +0530
Subject: [PATCH 06/11] update starship-action version to include more log
 lines

---
 .github/workflows/smoke-tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml
index ecb6c1d9b..d0d2b5692 100644
--- a/.github/workflows/smoke-tests.yaml
+++ b/.github/workflows/smoke-tests.yaml
@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.15-rc2
+        uses: cosmology-tech/starship-action@0.2.16-rc0
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -70,7 +70,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.15-rc2
+        uses: cosmology-tech/starship-action@0.2.16-rc0
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -107,7 +107,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.15-rc2
+        uses: cosmology-tech/starship-action@0.2.16-rc0
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true

From 457a33e4935b33cdb6150d0200a5ab4c75b86a93 Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 16:58:06 +0530
Subject: [PATCH 07/11] update logic for deleting home dir contents in the
 process of re-initialization

---
 charts/devnet/templates/chains/cosmos/validator.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index fe41162e3..b6f4d0fe4 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -111,13 +111,13 @@ spec:
               echo "Genesis file that we got....."
 
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
-                diff "$file1" "$file2" > /dev/null
+                diff "$CHAIN_DIR/config/genesis.json" "/tmp/genesis.json" > /dev/null
                 if [ $? -eq 0 ]; then
                   echo "Genesis file exists, exiting early"
                   exit 0
                 else
                   echo "Genesis found does not match..., performing cleanup"
-                  rm -rf $CHAIN_DIR
+                  rm -rf $CHAIN_DIR/*
                 fi
               fi
 

From 1198c143efda0ece96d5c7b9680791ca1c690e64 Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Wed, 30 Aug 2023 18:45:34 +0530
Subject: [PATCH 08/11] use hash of starship-action with more logs

---
 .github/workflows/smoke-tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml
index d0d2b5692..ae98f128f 100644
--- a/.github/workflows/smoke-tests.yaml
+++ b/.github/workflows/smoke-tests.yaml
@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.16-rc0
+        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -70,7 +70,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.16-rc0
+        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -107,7 +107,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@0.2.16-rc0
+        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true

From 17b4f5ec61e2edc1afbd034801f5842211c4765c Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Thu, 31 Aug 2023 08:19:03 +0530
Subject: [PATCH 09/11] add set -e to init container, use cmp instead of diff

---
 .github/workflows/smoke-tests.yaml                   | 2 +-
 charts/devnet/templates/chains/cosmos/validator.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml
index ae98f128f..21d492489 100644
--- a/.github/workflows/smoke-tests.yaml
+++ b/.github/workflows/smoke-tests.yaml
@@ -12,7 +12,7 @@ on:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-smoke-tests
-  cancel-in-progress: true
+  cancel-in-progress: false # debuging
 
 jobs:
   smoke-test-1:
diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index b6f4d0fe4..ec54f2796 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -101,6 +101,7 @@ spec:
             - bash
             - "-c"
             - |
+              set -e
               VAL_INDEX=${HOSTNAME##*-}
               echo "Validator Index: $VAL_INDEX"
               {{- if $toBuild }}
@@ -111,8 +112,7 @@ spec:
               echo "Genesis file that we got....."
 
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
-                diff "$CHAIN_DIR/config/genesis.json" "/tmp/genesis.json" > /dev/null
-                if [ $? -eq 0 ]; then
+                if cmp -s $CHAIN_DIR/config/genesis.json /tmp/genesis.json; then
                   echo "Genesis file exists, exiting early"
                   exit 0
                 else

From 88d010024fba32a437cfd37fa6f331d002a381ae Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Thu, 31 Aug 2023 16:15:39 +0530
Subject: [PATCH 10/11] use starship-action version with updated kind action

---
 .github/workflows/smoke-tests.yaml                   | 6 +++---
 charts/devnet/templates/chains/cosmos/validator.yaml | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/smoke-tests.yaml b/.github/workflows/smoke-tests.yaml
index 21d492489..6f57226f0 100644
--- a/.github/workflows/smoke-tests.yaml
+++ b/.github/workflows/smoke-tests.yaml
@@ -33,7 +33,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
+        uses: cosmology-tech/starship-action@5fe65293da878c468d1669061052680b4219fe37
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -70,7 +70,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
+        uses: cosmology-tech/starship-action@5fe65293da878c468d1669061052680b4219fe37
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
@@ -107,7 +107,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: cosmology-tech/starship-action@9c80712239104ea08c62424aec5ad0f875203fb4
+        uses: cosmology-tech/starship-action@5fe65293da878c468d1669061052680b4219fe37
         with:
           values: ${{ env.CONFIG_FILE }}
           port-forward: true
diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index ec54f2796..bf41073cd 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -110,6 +110,10 @@ spec:
 
               curl http://$GENESIS_HOST.$NAMESPACE.svc.cluster.local:$GENESIS_PORT/genesis -o /tmp/genesis.json
               echo "Genesis file that we got....."
+              echo "check tmp genesis"
+              ls -lhG /tmp/genesis.json
+              echo "check chain dir"
+              ls -lhG $CHAIN_DIR
 
               if [ -f $CHAIN_DIR/config/genesis.json ]; then
                 if cmp -s $CHAIN_DIR/config/genesis.json /tmp/genesis.json; then

From 5d24c06c8e547c61cc8090741b1c9f43ae01c524 Mon Sep 17 00:00:00 2001
From: Anmol1696 <anmol1696@gmail.com>
Date: Thu, 31 Aug 2023 16:46:01 +0530
Subject: [PATCH 11/11] remove strict exit, since it just fails

---
 charts/devnet/templates/chains/cosmos/validator.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/charts/devnet/templates/chains/cosmos/validator.yaml b/charts/devnet/templates/chains/cosmos/validator.yaml
index bf41073cd..2aeeaacc6 100644
--- a/charts/devnet/templates/chains/cosmos/validator.yaml
+++ b/charts/devnet/templates/chains/cosmos/validator.yaml
@@ -101,7 +101,6 @@ spec:
             - bash
             - "-c"
             - |
-              set -e
               VAL_INDEX=${HOSTNAME##*-}
               echo "Validator Index: $VAL_INDEX"
               {{- if $toBuild }}