Merge with upstream

.circleci/config.yml

@@ -1,14 +1,14 @@
 version: 2.1
 
 orbs:
-  go: circleci/go@1.7.3
+  go: circleci/go@1.12.0
   slack: circleci/slack@4.7.1
   codecov: codecov/codecov@3.2.4
 
 parameters:
   ubuntu_image:
     type: string
-    default: "ubuntu-2204:2022.04.2"
+    default: "ubuntu-2404:2024.05.1"
 
 workflows:
   version: 2
@@ -18,7 +18,7 @@ workflows:
           name: test_with_go_<< matrix.go_version >>
           matrix: &go-version-matrix
             parameters:
-              go_version: ["1.21.10"]
+              go_version: ["1.23.3"]
 
   circleci_build_and_test_nightly:
     triggers:
@@ -33,7 +33,7 @@ workflows:
           context: lamprey-secrets
           matrix: &go-version-matrix
             parameters:
-              go_version: ["1.21.10"]
+              go_version: ["1.23.3"]
       - indexer_vs_algod_nightly:
           name: nightly_test_indexer_vs_algod
           context: lamprey-secrets
@@ -135,7 +135,7 @@ commands:
     steps:
       - run:
           name: Install golangci-lint
-          command: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
+          command: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.0
 
   run_e2e_tests:
     steps:

.github/workflows/goreleaser.yml

@@ -12,13 +12,13 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.5.3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - run: git fetch --force --tags
 
       - name: go dependency
-        uses: actions/setup-go@v4.0.1
+        uses: actions/setup-go@v5
         with:
           go-version-file: 'go.mod'
 

.github/workflows/pr-type-category.yml

@@ -10,7 +10,7 @@ jobs:
     name: Check PR Category and Type
     steps:
       - name: Checking for correct number of required github pr labels
-        uses: mheap/github-action-required-labels@v2
+        uses: mheap/github-action-required-labels@v5
         with:
           mode: exactly
           count: 1

.github/workflows/reviewdog.yml

@@ -7,18 +7,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
       - name: Install specific golang
-        uses: actions/setup-go@v4.0.1
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.21.10'
+          go-version: '1.23.3'
       - name: reviewdog-golangci-lint
-        uses: reviewdog/action-golangci-lint@v2.6.1
+        uses: reviewdog/action-golangci-lint@v2.6.2
         with:
           go_version_file: go.mod
-          golangci_lint_version: "v1.58.0"
+          golangci_lint_version: "v1.62.0"
           golangci_lint_flags: "-c .golangci.yml --allow-parallel-runners"
           reporter: "github-pr-check"
           tool_name: "Lint Errors"
@@ -30,18 +30,18 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0 # required for new-from-rev option in .golangci.yml
       - name: Install specific golang
-        uses: actions/setup-go@v4.0.1
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.21.10'
+          go-version: '1.23.3'
       - name: reviewdog-golangci-lint
-        uses: reviewdog/action-golangci-lint@v2.6.1
+        uses: reviewdog/action-golangci-lint@v2.6.2
         with:
           go_version_file: go.mod
-          golangci_lint_version: "v1.58.0"
+          golangci_lint_version: "v1.62.0"
           golangci_lint_flags: "-c .golangci-warnings.yml --allow-parallel-runners"
           reporter: "github-pr-check"
           tool_name: "Lint Warnings"

.golangci.yml

@@ -1,6 +1,6 @@
 run:
   timeout: 5m
-  tests: false
+  tests: true
 
 linters:
   disable-all: true
@@ -66,4 +66,4 @@ issues:
 
     # revive: ignore some rules
     - "^unused-parameter: parameter"
-    - "^package-comments: should have a package comment"
+    - "^package-comments: should have a package comment"

Makefile

@@ -20,7 +20,7 @@ idb/postgres/internal/schema/setup_postgres_sql.go:	idb/postgres/internal/schema
 	cd idb/postgres/internal/schema && go generate
 
 idb/mocks/IndexerDb.go:	idb/idb.go
-	go install github.com/vektra/mockery/v2@v2.20.0
+	go install github.com/vektra/mockery/v2@v2.47.0
 	cd idb && mockery --name=IndexerDb
 
 # check that all packages (except tests) compile

accounting/rewind.go

@@ -0,0 +1,178 @@
+package accounting
+
+import (
+	"context"
+	"fmt"
+
+	models "github.com/algorand/indexer/v3/api/generated/v2"
+	"github.com/algorand/indexer/v3/idb"
+	"github.com/algorand/indexer/v3/types"
+
+	sdk "github.com/algorand/go-algorand-sdk/v2/types"
+)
+
+// ConsistencyError is returned when the database returns inconsistent (stale) results.
+type ConsistencyError struct {
+	msg string
+}
+
+func (e ConsistencyError) Error() string {
+	return e.msg
+}
+func assetUpdate(account *models.Account, assetid uint64, add, sub uint64) {
+	if account.Assets == nil {
+		account.Assets = new([]models.AssetHolding)
+	}
+	assets := *account.Assets
+	for i, ah := range assets {
+		if ah.AssetId == assetid {
+			ah.Amount += add
+			ah.Amount -= sub
+			assets[i] = ah
+			// found and updated asset, done
+			return
+		}
+	}
+	// add asset to list
+	assets = append(assets, models.AssetHolding{
+		Amount:  add - sub,
+		AssetId: assetid,
+		//Creator: base32 addr string of asset creator, TODO
+		//IsFrozen: leave nil? // TODO: on close record frozen state for rewind
+	})
+	*account.Assets = assets
+}
+
+// SpecialAccountRewindError indicates that an attempt was made to rewind one of the special accounts.
+type SpecialAccountRewindError struct {
+	account string
+}
+
+// MakeSpecialAccountRewindError helper to initialize a SpecialAccountRewindError.
+func MakeSpecialAccountRewindError(account string) *SpecialAccountRewindError {
+	return &SpecialAccountRewindError{account: account}
+}
+
+// Error is part of the error interface.
+func (sare *SpecialAccountRewindError) Error() string {
+	return fmt.Sprintf("unable to rewind the %s", sare.account)
+}
+
+var specialAccounts *types.SpecialAddresses
+
+// AccountAtRound queries the idb.IndexerDb object for transactions and rewinds most fields of the account back to
+// their values at the requested round.
+// `round` must be <= `account.Round`
+func AccountAtRound(ctx context.Context, account models.Account, round uint64, db idb.IndexerDb) (acct models.Account, err error) {
+	// Make sure special accounts cache has been initialized.
+	if specialAccounts == nil {
+		var accounts types.SpecialAddresses
+		accounts, err = db.GetSpecialAccounts(ctx)
+		if err != nil {
+			return models.Account{}, fmt.Errorf("unable to get special accounts: %v", err)
+		}
+		specialAccounts = &accounts
+	}
+	acct = account
+	var addr sdk.Address
+	addr, err = sdk.DecodeAddress(account.Address)
+	if err != nil {
+		return
+	}
+	// ensure that the don't attempt to rewind a special account.
+	if specialAccounts.FeeSink == addr {
+		err = MakeSpecialAccountRewindError("FeeSink")
+		return
+	}
+	if specialAccounts.RewardsPool == addr {
+		err = MakeSpecialAccountRewindError("RewardsPool")
+		return
+	}
+	// Get transactions and rewind account.
+	tf := idb.TransactionFilter{
+		Address:  addr[:],
+		MinRound: round + 1,
+		MaxRound: account.Round,
+	}
+	ctx2, cf := context.WithCancel(ctx)
+	// In case of a panic before the next defer, call cf() here.
+	defer cf()
+	txns, r := db.Transactions(ctx2, tf)
+	// In case of an error, make sure the context is cancelled, and the channel is cleaned up.
+	defer func() {
+		cf()
+		for range txns {
+		}
+	}()
+	if r < account.Round {
+		err = ConsistencyError{fmt.Sprintf("queried round r: %d < account.Round: %d", r, account.Round)}
+		return
+	}
+	txcount := 0
+	for txnrow := range txns {
+		if txnrow.Error != nil {
+			err = txnrow.Error
+			return
+		}
+		txcount++
+		stxn := txnrow.Txn
+		if stxn == nil {
+			return models.Account{},
+				fmt.Errorf("rewinding past inner transactions is not supported")
+		}
+		if addr == stxn.Txn.Sender {
+			acct.AmountWithoutPendingRewards += uint64(stxn.Txn.Fee)
+			acct.AmountWithoutPendingRewards -= uint64(stxn.SenderRewards)
+		}
+		switch stxn.Txn.Type {
+		case sdk.PaymentTx:
+			if addr == stxn.Txn.Sender {
+				acct.AmountWithoutPendingRewards += uint64(stxn.Txn.Amount)
+			}
+			if addr == stxn.Txn.Receiver {
+				acct.AmountWithoutPendingRewards -= uint64(stxn.Txn.Amount)
+				acct.AmountWithoutPendingRewards -= uint64(stxn.ReceiverRewards)
+			}
+			if addr == stxn.Txn.CloseRemainderTo {
+				// unwind receiving a close-to
+				acct.AmountWithoutPendingRewards -= uint64(stxn.ClosingAmount)
+				acct.AmountWithoutPendingRewards -= uint64(stxn.CloseRewards)
+			} else if !stxn.Txn.CloseRemainderTo.IsZero() {
+				// unwind sending a close-to
+				acct.AmountWithoutPendingRewards += uint64(stxn.ClosingAmount)
+			}
+		case sdk.KeyRegistrationTx:
+			// TODO: keyreg does not rewind. workaround: query for txns on an account with typeenum=2 to find previous values it was set to.
+		case sdk.AssetConfigTx:
+			if stxn.Txn.ConfigAsset == 0 {
+				// create asset, unwind the application of the value
+				assetUpdate(&acct, txnrow.AssetID, 0, stxn.Txn.AssetParams.Total)
+			}
+		case sdk.AssetTransferTx:
+			if addr == stxn.Txn.AssetSender || addr == stxn.Txn.Sender {
+				assetUpdate(&acct, uint64(stxn.Txn.XferAsset), stxn.Txn.AssetAmount+txnrow.Extra.AssetCloseAmount, 0)
+			}
+			if addr == stxn.Txn.AssetReceiver {
+				assetUpdate(&acct, uint64(stxn.Txn.XferAsset), 0, stxn.Txn.AssetAmount)
+			}
+			if addr == stxn.Txn.AssetCloseTo {
+				assetUpdate(&acct, uint64(stxn.Txn.XferAsset), 0, txnrow.Extra.AssetCloseAmount)
+			}
+		case sdk.AssetFreezeTx:
+		default:
+			err = fmt.Errorf("%s[%d,%d]: rewinding past txn type %s is not currently supported", account.Address, txnrow.Round, txnrow.Intra, stxn.Txn.Type)
+			return
+		}
+	}
+	acct.Round = round
+	// Due to accounts being closed and re-opened, we cannot always rewind Rewards. So clear it out.
+	acct.Rewards = 0
+	// Computing pending rewards is not supported.
+	acct.PendingRewards = 0
+	acct.Amount = acct.AmountWithoutPendingRewards
+	// MinBalance is not supported.
+	acct.MinBalance = 0
+	// TODO: Clear out the closed-at field as well. Like Rewards we cannot know this value for all accounts.
+	//acct.ClosedAt = 0
+	return
+}

api/README.md

@@ -9,7 +9,7 @@ The API is defined using [OpenAPI v2](https://swagger.io/specification/v2/) in *
 The Makefile will install our fork of **oapi-codegen**, use `make oapi-codegen` to install it directly.
 
 1. Document your changes by editing **indexer.oas2.yml**
-2. Regenerate the endpoints by running **generate.sh**. The sources at **generated/** will be updated.
+2. Regenerate the endpoints by running **make generate** from the `api` directory. The sources at **generated/** will be updated.
 3. Update the implementation in **handlers.go**. It is sometimes useful to consult **generated/routes.go** to make sure the handler properly implements **ServerInterface**.
 
 ## What codegen tool is used?

api/app_boxes_fixtures_test.go

@@ -6,13 +6,14 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/algorand/indexer/v3/idb/postgres"
 	"github.com/stretchr/testify/require"
 
 	"github.com/algorand/avm-abi/apps"
+	"github.com/algorand/indexer/v3/idb/postgres"
+	"github.com/algorand/indexer/v3/util/test"
+
 	"github.com/algorand/go-algorand-sdk/v2/crypto"
 	sdk "github.com/algorand/go-algorand-sdk/v2/types"
-	"github.com/algorand/indexer/v3/util/test"
 )
 
 func goalEncode(t *testing.T, s string) string {
@@ -23,7 +24,7 @@ func goalEncode(t *testing.T, s string) string {
 	return string(b2)
 }
 
-var goalEncodingExamples map[string]string = map[string]string{
+var goalEncodingExamples = map[string]string{
 	"str":         "str",
 	"string":      "string",
 	"int":         "42",