major: breaking changes, fixes

* add global lock
* fix lock contention
* update redis client and server -> v7
* add network status API
* upgrade deps

Squashed commit of the following:

commit 9d95f2e
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Thu Mar 16 11:12:06 2023 +0000

    feat: add updated_at postgres fn + trigger

    * closes #66

commit 144d501
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Thu Mar 16 08:48:45 2023 +0000

    feat: add network account status (nonce, balance)

commit 5679a67
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Thu Mar 16 08:01:58 2023 +0000

    fix: system global lock

    * add middleware to entire API group
    * setNX system lock key

commit ee907dd
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Thu Mar 16 07:34:29 2023 +0000

    feat: add system global lock to be triggered manually

commit ad58d1d
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Tue Mar 14 16:02:25 2023 +0000

    feat: add lock retry strategy

    * previouly we relied on the task being re-queued which generally reduces the throughput of tasks

commit f4e3aed
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Tue Mar 14 15:45:06 2023 +0000

    tasker: add support for reporting panics

commit b8ebf88
Author: Mohammed Sohail <sohailsameja@gmail.com>
Date:   Tue Mar 14 15:40:50 2023 +0000

    pkg: bump go-redis -> v9

commit 4a0bf88
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Date:   Tue Mar 14 18:15:05 2023 +0300

    build(deps): bump github.com/jackc/tern/v2 from 2.0.0 to 2.0.1 (#69)

    Bumps [github.com/jackc/tern/v2](https://github.com/jackc/tern) from 2.0.0 to 2.0.1.
    - [Release notes](https://github.com/jackc/tern/releases)
    - [Changelog](https://github.com/jackc/tern/blob/master/.goreleaser.yaml)
    - [Commits](jackc/tern@v2.0.0...v2.0.1)

    ---
    updated-dependencies:
    - dependency-name: github.com/jackc/tern/v2
      dependency-type: direct:production
      update-type: version-update:semver-patch
    ...

    Signed-off-by: dependabot[bot] <support@github.com>
    Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

commit 5328d27
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Date:   Tue Mar 14 18:14:45 2023 +0300

    build(deps): bump golang.org/x/crypto from 0.6.0 to 0.7.0 (#70)

    Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.6.0 to 0.7.0.
    - [Release notes](https://github.com/golang/crypto/releases)
    - [Commits](golang/crypto@v0.6.0...v0.7.0)

    ---
    updated-dependencies:
    - dependency-name: golang.org/x/crypto
      dependency-type: direct:production
      update-type: version-update:semver-minor
    ...

    Signed-off-by: dependabot[bot] <support@github.com>
    Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

commit 7ce80f9
Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Date:   Tue Mar 14 18:13:59 2023 +0300

    build(deps): bump github.com/grassrootseconomics/celoutils (#71)

    Bumps [github.com/grassrootseconomics/celoutils](https://github.com/grassrootseconomics/celoutils) from 1.0.0 to 1.1.1.
    - [Release notes](https://github.com/grassrootseconomics/celoutils/releases)
    - [Commits](grassrootseconomics/celoutils@v1.0.0...v1.1.1)

    ---
    updated-dependencies:
    - dependency-name: github.com/grassrootseconomics/celoutils
      dependency-type: direct:production
      update-type: version-update:semver-minor
    ...

    Signed-off-by: dependabot[bot] <support@github.com>
    Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/release.yaml

@@ -38,7 +38,7 @@ jobs:
 
     - name: Set outputs
       run: |
-        echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+        echo "RELEASE_TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV \
         echo "RELEASE_SHORT_COMMIT=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
     - name: Build and push image

.gitignore

@@ -1,4 +1,4 @@
 cic-custodial
 dist
-.env
+**/*.env
 covprofile

cmd/service/api.go

@@ -13,7 +13,8 @@ import (
 )
 
 const (
-	contextTimeout = 5 * time.Second
+	contextTimeout      = 5 * time.Second
+	systemGlobalLockKey = "system:global_lock"
 )
 
 // Bootstrap API server.
@@ -46,8 +47,10 @@ func initApiServer(custodialContainer *custodial.Custodial) *echo.Echo {
 		})
 	}
 
-	apiRoute := server.Group("/api")
+	apiRoute := server.Group("/api", systemGlobalLock)
+
 	apiRoute.POST("/account/create", api.HandleAccountCreate)
+	apiRoute.GET("/account/status/:address", api.HandleNetworkAccountStatus)
 	apiRoute.POST("/sign/transfer", api.HandleSignTransfer)
 	apiRoute.GET("/track/:trackingId", api.HandleTrackTx)
 
@@ -81,3 +84,25 @@ func customHTTPErrorHandler(err error, c echo.Context) {
 		Message: "Internal server error.",
 	})
 }
+
+func systemGlobalLock(next echo.HandlerFunc) echo.HandlerFunc {
+	return func(c echo.Context) error {
+		var (
+			cu = c.Get("cu").(*custodial.Custodial)
+		)
+
+		locked, err := cu.RedisClient.Get(c.Request().Context(), systemGlobalLockKey).Bool()
+		if err != nil {
+			return err
+		}
+
+		if locked {
+			return c.JSON(http.StatusServiceUnavailable, api.ErrResp{
+				Ok:      false,
+				Message: "System manually locked.",
+			})
+		}
+
+		return next(c)
+	}
+}

cmd/service/custodial.go

@@ -6,10 +6,10 @@ import (
 	"time"
 
 	eth_crypto "github.com/celo-org/celo-blockchain/crypto"
-	"github.com/go-redis/redis/v8"
 	"github.com/grassrootseconomics/cic-custodial/internal/custodial"
 	"github.com/grassrootseconomics/cic-custodial/internal/nonce"
 	"github.com/grassrootseconomics/w3-celo-patch"
+	"github.com/redis/go-redis/v9"
 )
 
 // Define common smart contrcat ABI's that can be injected into the system container.

cmd/service/tasker.go

@@ -72,6 +72,8 @@ func observibilityMiddleware() asynq.MiddlewareFunc {
 			err := handler.ProcessTask(ctx, task)
 			if err != nil && isFailureHandler(err) {
 				lo.Error("tasker: handler error", "err", err, "task_type", task.Type(), "task_id", taskId)
+			} else if asynq.IsPanicError(err) {
+				lo.Error("tasker: handler panic", "err", err, "task_type", task.Type(), "task_id", taskId)
 			} else {
 				lo.Info("tasker: process task", "task_type", task.Type(), "task_id", taskId)
 			}

dev/docker-compose.dev.yaml → dev/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   redis:
-    image: redis:6-alpine
+    image: redis:7-alpine
     restart: unless-stopped
     command: redis-server --save 60 1 --loglevel warning
     volumes:
@@ -20,9 +20,9 @@ services:
     environment:
       - POSTGRES_PASSWORD=postgres
       - POSTGRES_USER=postgres
-      - POSTGRES_DB=cic_custodial
     volumes:
       - cic-custodial-pg:/var/lib/postgresql/data
+      - ./init_db.sql:/docker-entrypoint-initdb.d/init_db.sql
     ports:
       - "127.0.0.1:5432:5432"
     healthcheck:
@@ -37,8 +37,8 @@ services:
     volumes:
       - cic-custodial-nats:/nats/data
     ports:
-      - "4222:4222"
-      - "8222:8222"
+      - "127.0.0.1:4222:4222"
+      - "127.0.0.1:8222:8222"
   asynqmon:
     image: hibiken/asynqmon
     restart: unless-stopped
@@ -49,6 +49,16 @@ services:
     depends_on:
       redis:
         condition: service_healthy
+  cic-chain-events:
+    image: ghcr.io/grassrootseconomics/cic-chain-events/cic-chain-events:latest
+    restart: unless-stopped    
+    env_file:
+      - events.env
+    ports:
+      - '127.0.0.1:5001:5000'
+    depends_on:
+      postgres:
+        condition: service_healthy
 volumes:
   cic-custodial-pg:
     driver: local

dev/init_db.sql

@@ -0,0 +1,2 @@
+CREATE DATABASE cic_custodial;
+CREATE DATABASE cic_chain_events;

go.mod

@@ -4,28 +4,30 @@ go 1.20
 
 require (
 	github.com/VictoriaMetrics/metrics v1.23.1
-	github.com/bsm/redislock v0.7.2
+	github.com/bsm/redislock v0.9.1
 	github.com/celo-org/celo-blockchain v1.7.2
 	github.com/georgysavva/scany/v2 v2.0.0
 	github.com/go-playground/validator/v10 v10.11.2
-	github.com/go-redis/redis/v8 v8.11.5
 	github.com/google/uuid v1.3.0
-	github.com/grassrootseconomics/celoutils v1.0.0
+	github.com/grassrootseconomics/celoutils v1.1.1
 	github.com/grassrootseconomics/w3-celo-patch v0.2.0
 	github.com/hibiken/asynq v0.24.0
 	github.com/jackc/pgx/v5 v5.3.1
-	github.com/jackc/tern/v2 v2.0.0
+	github.com/jackc/tern/v2 v2.0.1
 	github.com/knadh/goyesql/v2 v2.2.0
 	github.com/knadh/koanf/parsers/toml v0.1.0
 	github.com/knadh/koanf/providers/env v0.1.0
 	github.com/knadh/koanf/providers/file v0.1.0
 	github.com/knadh/koanf/v2 v2.0.0
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/nats-io/nats.go v1.24.0
+	github.com/redis/go-redis/v9 v9.0.2
 	github.com/zerodha/logf v0.5.5
-	golang.org/x/crypto v0.6.0
+	golang.org/x/crypto v0.7.0
 )
 
+replace github.com/hibiken/asynq => github.com/grassrootseconomics/asynq v0.25.0
+
 require (
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
@@ -40,7 +42,7 @@ require (
 	github.com/celo-org/celo-bls-go-macos v0.6.3 // indirect
 	github.com/celo-org/celo-bls-go-other v0.6.3 // indirect
 	github.com/celo-org/celo-bls-go-windows v0.6.3 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -77,6 +79,7 @@ require (
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/ginkgo v1.16.4 // indirect
 	github.com/onsi/gomega v1.24.1 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -94,10 +97,10 @@ require (
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/net v0.8.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect