feat: add message_type module and store msg types inside message_type table

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Unreleased
 - ([\#610](https://github.com/forbole/bdjuno/pull/610)) Add support for gov `v1` proposals
 - ([\#652](https://github.com/forbole/bdjuno/pull/652)) Update gov module parsing 
+- ([\#702](https://github.com/forbole/bdjuno/pull/702)) Add `message_type` module and store msg types inside `message_type` table, add `messages_by_type` function to allow to query messages by their types
 
 ## Version v4.0.0
 ## Notes

cmd/migrate/cmd.go

@@ -8,13 +8,15 @@ import (
 	"github.com/spf13/cobra"
 
 	v3 "github.com/forbole/bdjuno/v4/cmd/migrate/v3"
+	v5 "github.com/forbole/bdjuno/v4/cmd/migrate/v5"
 )
 
 type Migrator func(parseCfg *parsecmdtypes.Config) error
 
 var (
 	migrations = map[string]Migrator{
 		"v3": v3.RunMigration,
+		"v5": v5.RunMigration,
 	}
 )
 

cmd/migrate/v5/migrate.go

@@ -0,0 +1,43 @@
+package v5
+
+import (
+	"fmt"
+
+	v5db "github.com/forbole/bdjuno/v4/database/migrate/v5"
+	parse "github.com/forbole/juno/v5/cmd/parse/types"
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+	"github.com/forbole/juno/v5/types/config"
+)
+
+// RunMigration runs the migrations to v5
+func RunMigration(parseConfig *parse.Config) error {
+	cfg, err := GetConfig()
+	if err != nil {
+		return fmt.Errorf("error while reading config: %s", err)
+	}
+
+	// Migrate the database
+	err = migrateDb(cfg, parseConfig)
+	if err != nil {
+		return fmt.Errorf("error while migrating database: %s", err)
+	}
+
+	return nil
+}
+
+func migrateDb(cfg config.Config, parseConfig *parse.Config) error {
+	// Build the codec
+	encodingConfig := parseConfig.GetEncodingConfigBuilder()()
+
+	// Get the db
+	databaseCtx := database.NewContext(cfg.Database, encodingConfig, parseConfig.GetLogger())
+	db, err := postgresql.Builder(databaseCtx)
+	if err != nil {
+		return fmt.Errorf("error while building the db: %s", err)
+	}
+
+	// Build the migrator and perform the migrations
+	migrator := v5db.NewMigrator(db.(*postgresql.Database))
+	return migrator.Migrate()
+}

cmd/migrate/v5/utils.go

@@ -0,0 +1,29 @@
+package v5
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/forbole/juno/v5/types/config"
+	"gopkg.in/yaml.v3"
+)
+
+// GetConfig returns the configuration reading it from the config.yaml file present inside the home directory
+func GetConfig() (config.Config, error) {
+	file := path.Join(config.HomePath, "config.yaml")
+
+	// Make sure the path exists
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		return config.Config{}, fmt.Errorf("config file does not exist")
+	}
+
+	bz, err := os.ReadFile(file)
+	if err != nil {
+		return config.Config{}, fmt.Errorf("error while reading config file: %s", err)
+	}
+
+	var cfg config.Config
+	err = yaml.Unmarshal(bz, &cfg)
+	return cfg, err
+}

database/database_test.go

@@ -24,7 +24,8 @@ import (
 	tmtypes "github.com/cometbft/cometbft/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
-	simappparams "cosmossdk.io/simapp/params"
+	"github.com/forbole/juno/v5/types/params"
+
 	"github.com/stretchr/testify/suite"
 
 	_ "github.com/proullon/ramsql/driver"
@@ -42,7 +43,7 @@ type DbTestSuite struct {
 
 func (suite *DbTestSuite) SetupTest() {
 	// Create the codec
-	codec := simappparams.MakeTestEncodingConfig()
+	codec := params.MakeTestEncodingConfig()
 
 	// Build the database
 	dbCfg := dbconfig.NewDatabaseConfig(
@@ -56,7 +57,7 @@ func (suite *DbTestSuite) SetupTest() {
 		100000,
 		100,
 	)
-	db, err := database.Builder(junodb.NewContext(dbCfg, &codec, logging.DefaultLogger()))
+	db, err := database.Builder(junodb.NewContext(dbCfg, codec, logging.DefaultLogger()))
 	suite.Require().NoError(err)
 
 	bigDipperDb, ok := (db).(*database.Db)

database/message_type.go

@@ -0,0 +1,16 @@
+package database
+
+import (
+	types "github.com/forbole/bdjuno/v4/types"
+)
+
+// SaveMessageType stores the given message type inside the database
+func (db *Db) SaveMessageType(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrate.go

@@ -0,0 +1,53 @@
+package v5
+
+import (
+	"fmt"
+
+	utils "github.com/forbole/bdjuno/v4/modules/utils"
+	"github.com/forbole/bdjuno/v4/types"
+)
+
+// Migrate implements database.Migrator
+func (db *Migrator) Migrate() error {
+	msgTypes, err := db.getMsgTypesFromMessageTable()
+	if err != nil {
+		return fmt.Errorf("error while getting message types rows: %s", err)
+	}
+
+	for _, msgType := range msgTypes {
+		// migrate message types
+		err = db.migrateMsgTypes(types.NewMessageType(
+			msgType.Type,
+			utils.GetModuleNameFromTypeURL(msgType.Type),
+			utils.GetMsgFromTypeURL(msgType.Type),
+			msgType.Height))
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getMsgTypesFromMessageTable retrieves messages types stored in database inside message table
+func (db *Migrator) getMsgTypesFromMessageTable() ([]MessageRow, error) {
+	smt := "SELECT DISTINCT ON (type) type, transaction_hash, height FROM message ORDER BY type DESC"
+	var rows []MessageRow
+	err := db.SQL.Select(&rows, smt)
+	if err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+// migrateMsgTypes stores the given message type inside the database
+func (db *Migrator) migrateMsgTypes(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrator.go

@@ -0,0 +1,21 @@
+package v5
+
+import (
+	"github.com/jmoiron/sqlx"
+
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+)
+
+var _ database.Migrator = &Migrator{}
+
+// Migrator represents the database migrator that should be used to migrate from v4 of the database to v5
+type Migrator struct {
+	SQL *sqlx.DB
+}
+
+func NewMigrator(db *postgresql.Database) *Migrator {
+	return &Migrator{
+		SQL: db.SQL,
+	}
+}

database/migrate/v5/types.go

@@ -0,0 +1,10 @@
+package v5
+
+type MessageRow struct {
+	TransactionHash           string `db:"transaction_hash"`
+	Index                     int64  `db:"index"`
+	Type                      string `db:"type"`
+	Value                     string `db:"value"`
+	InvolvedAccountsAddresses string `db:"involved_accounts_addresses"`
+	Height                    int64  `db:"height"`
+}

database/schema/00-cosmos.sql

@@ -66,11 +66,21 @@ CREATE INDEX transaction_hash_index ON transaction (hash);
 CREATE INDEX transaction_height_index ON transaction (height);
 CREATE INDEX transaction_partition_id_index ON transaction (partition_id);
 
+CREATE TABLE message_type
+(
+    type      TEXT   NOT NULL UNIQUE,
+    module    TEXT   NOT NULL,
+    label     TEXT   NOT NULL,
+    height    BIGINT NOT NULL
+);
+CREATE INDEX message_type_module_index ON message_type (module);
+CREATE INDEX message_type_type_index ON message_type (type);
+
 CREATE TABLE message
 (
     transaction_hash            TEXT   NOT NULL,
     index                       BIGINT NOT NULL,
-    type                        TEXT   NOT NULL,
+    type                        TEXT   NOT NULL REFERENCES message_type(type),
     value                       JSONB  NOT NULL,
     involved_accounts_addresses TEXT[] NOT NULL,
 
@@ -101,6 +111,17 @@ WHERE (cardinality(types) = 0 OR type = ANY (types))
 ORDER BY height DESC LIMIT "limit" OFFSET "offset"
 $$ LANGUAGE sql STABLE;
 
+CREATE FUNCTION messages_by_type(
+  types text [],
+  "limit" bigint DEFAULT 100,
+  "offset" bigint DEFAULT 0) 
+  RETURNS SETOF message AS 
+$$ 
+SELECT * FROM message
+WHERE (cardinality(types) = 0 OR type = ANY (types))
+ORDER BY height DESC LIMIT "limit" OFFSET "offset" 
+$$ LANGUAGE sql STABLE;
+
 CREATE TABLE pruning
 (
     last_pruned_height BIGINT NOT NULL

go.mod

@@ -4,11 +4,11 @@ go 1.20
 
 require (
 	cosmossdk.io/math v1.2.0
-	cosmossdk.io/simapp v0.0.0-20230224204036-a6adb0821462
+	cosmossdk.io/simapp v0.0.0-20230712090904-031162fbb96e
 	github.com/cometbft/cometbft v0.37.2
-	github.com/cosmos/cosmos-sdk v0.47.2
+	github.com/cosmos/cosmos-sdk v0.47.4
 	github.com/cosmos/gogoproto v1.4.10
-	github.com/forbole/juno/v5 v5.2.0
+	github.com/forbole/juno/v5 v5.2.1-0.20240201075935-851426ddd905
 	github.com/go-co-op/gocron v1.37.0
 	github.com/golangci/golangci-lint v1.55.2
 	github.com/jmoiron/sqlx v1.3.5
@@ -35,7 +35,8 @@ require (
 	cosmossdk.io/api v0.3.1 // indirect
 	cosmossdk.io/core v0.5.1 // indirect
 	cosmossdk.io/depinject v1.0.0-alpha.3 // indirect
-	cosmossdk.io/errors v1.0.0-beta.7 // indirect
+	cosmossdk.io/errors v1.0.0 // indirect
+	cosmossdk.io/log v1.1.1-0.20230704160919-88f2c830b0ca // indirect
 	cosmossdk.io/tools/rosetta v0.2.1 // indirect
 	filippo.io/edwards25519 v1.0.0 // indirect
 	github.com/4meepo/tagalign v1.3.3 // indirect
@@ -220,8 +221,8 @@ require (
 	github.com/nishanths/predeclared v0.2.2 // indirect
 	github.com/nunnatsa/ginkgolinter v0.14.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
-	github.com/petermattis/goid v0.0.0-20221215004737-a150e88a970d // indirect
+	github.com/pelletier/go-toml/v2 v2.0.7 // indirect
+	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polyfloyd/go-errorlint v1.4.5 // indirect
@@ -320,4 +321,4 @@ require (
 
 // This is to avoid warnings while running the binary
 // See here: https://github.com/desmos-labs/desmos/pull/1131#discussion_r1194090419
-replace github.com/cosmos/gogoproto => github.com/cosmos/gogoproto v1.4.8
+replace github.com/cosmos/gogoproto => github.com/cosmos/gogoproto v1.4.10