feat: add message_type module and store msg types inside `message_t…

…ype` table (#702)

Closes: #XXXX

<!-- Add a description of the changes that this PR introduces and the
files that
are the most critical to review. -->

Changes:
- Added `message_type` module
- Added msg handler to store message types inside `message_type` table
- Added new `messages_by_type` function to allow to query messages by
their types
- Added `v5` migrator to parse all msg types already stored in db and
save them inside `message_type` table

---

*All items are required. Please add a note to the item if the item is
not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type
prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json)
in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch
- [x] provided a link to the relevant issue or specification
- [x] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go
code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [x] confirmed all CI checks have passed

*All items are required. Please add a note if the item is not applicable
and please add
your handle next to the items reviewed if you only reviewed selected
items.*

I have...

- [ ] confirmed the correct [type
prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json)
in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Unreleased
 - ([\#610](https://github.com/forbole/bdjuno/pull/610)) Add support for gov `v1` proposals
 - ([\#652](https://github.com/forbole/bdjuno/pull/652)) Update gov module parsing 
+- ([\#702](https://github.com/forbole/bdjuno/pull/702)) Add `message_type` module and store msg types inside `message_type` table, add `messages_by_type` function to allow to query messages by their types
 
 ## Version v4.0.0
 ## Notes

cmd/migrate/cmd.go

@@ -8,13 +8,15 @@ import (
 	"github.com/spf13/cobra"
 
 	v3 "github.com/forbole/bdjuno/v4/cmd/migrate/v3"
+	v5 "github.com/forbole/bdjuno/v4/cmd/migrate/v5"
 )
 
 type Migrator func(parseCfg *parsecmdtypes.Config) error
 
 var (
 	migrations = map[string]Migrator{
 		"v3": v3.RunMigration,
+		"v5": v5.RunMigration,
 	}
 )
 

cmd/migrate/v5/migrate.go

@@ -0,0 +1,43 @@
+package v5
+
+import (
+	"fmt"
+
+	v5db "github.com/forbole/bdjuno/v4/database/migrate/v5"
+	parse "github.com/forbole/juno/v5/cmd/parse/types"
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+	"github.com/forbole/juno/v5/types/config"
+)
+
+// RunMigration runs the migrations to v5
+func RunMigration(parseConfig *parse.Config) error {
+	cfg, err := GetConfig()
+	if err != nil {
+		return fmt.Errorf("error while reading config: %s", err)
+	}
+
+	// Migrate the database
+	err = migrateDb(cfg, parseConfig)
+	if err != nil {
+		return fmt.Errorf("error while migrating database: %s", err)
+	}
+
+	return nil
+}
+
+func migrateDb(cfg config.Config, parseConfig *parse.Config) error {
+	// Build the codec
+	encodingConfig := parseConfig.GetEncodingConfigBuilder()()
+
+	// Get the db
+	databaseCtx := database.NewContext(cfg.Database, encodingConfig, parseConfig.GetLogger())
+	db, err := postgresql.Builder(databaseCtx)
+	if err != nil {
+		return fmt.Errorf("error while building the db: %s", err)
+	}
+
+	// Build the migrator and perform the migrations
+	migrator := v5db.NewMigrator(db.(*postgresql.Database))
+	return migrator.Migrate()
+}

cmd/migrate/v5/utils.go

@@ -0,0 +1,29 @@
+package v5
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/forbole/juno/v5/types/config"
+	"gopkg.in/yaml.v3"
+)
+
+// GetConfig returns the configuration reading it from the config.yaml file present inside the home directory
+func GetConfig() (config.Config, error) {
+	file := path.Join(config.HomePath, "config.yaml")
+
+	// Make sure the path exists
+	if _, err := os.Stat(file); os.IsNotExist(err) {
+		return config.Config{}, fmt.Errorf("config file does not exist")
+	}
+
+	bz, err := os.ReadFile(file)
+	if err != nil {
+		return config.Config{}, fmt.Errorf("error while reading config file: %s", err)
+	}
+
+	var cfg config.Config
+	err = yaml.Unmarshal(bz, &cfg)
+	return cfg, err
+}

database/database_test.go

@@ -24,7 +24,8 @@ import (
 	tmtypes "github.com/cometbft/cometbft/types"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
-	params "github.com/forbole/juno/v5/types/params"
+	"github.com/forbole/juno/v5/types/params"
+
 	"github.com/stretchr/testify/suite"
 
 	_ "github.com/proullon/ramsql/driver"

database/message_type.go

@@ -0,0 +1,16 @@
+package database
+
+import (
+	types "github.com/forbole/bdjuno/v4/types"
+)
+
+// SaveMessageType stores the given message type inside the database
+func (db *Db) SaveMessageType(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrate.go

@@ -0,0 +1,53 @@
+package v5
+
+import (
+	"fmt"
+
+	utils "github.com/forbole/bdjuno/v4/modules/utils"
+	"github.com/forbole/bdjuno/v4/types"
+)
+
+// Migrate implements database.Migrator
+func (db *Migrator) Migrate() error {
+	msgTypes, err := db.getMsgTypesFromMessageTable()
+	if err != nil {
+		return fmt.Errorf("error while getting message types rows: %s", err)
+	}
+
+	for _, msgType := range msgTypes {
+		// migrate message types
+		err = db.migrateMsgTypes(types.NewMessageType(
+			msgType.Type,
+			utils.GetModuleNameFromTypeURL(msgType.Type),
+			utils.GetMsgFromTypeURL(msgType.Type),
+			msgType.Height))
+
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getMsgTypesFromMessageTable retrieves messages types stored in database inside message table
+func (db *Migrator) getMsgTypesFromMessageTable() ([]MessageRow, error) {
+	smt := "SELECT DISTINCT ON (type) type, transaction_hash, height FROM message ORDER BY type DESC"
+	var rows []MessageRow
+	err := db.SQL.Select(&rows, smt)
+	if err != nil {
+		return nil, err
+	}
+
+	return rows, nil
+}
+
+// migrateMsgTypes stores the given message type inside the database
+func (db *Migrator) migrateMsgTypes(msg *types.MessageType) error {
+	stmt := `
+INSERT INTO message_type(type, module, label, height) 
+VALUES ($1, $2, $3, $4) 
+ON CONFLICT (type) DO NOTHING`
+
+	_, err := db.SQL.Exec(stmt, msg.Type, msg.Module, msg.Label, msg.Height)
+	return err
+}

database/migrate/v5/migrator.go

@@ -0,0 +1,21 @@
+package v5
+
+import (
+	"github.com/jmoiron/sqlx"
+
+	"github.com/forbole/juno/v5/database"
+	"github.com/forbole/juno/v5/database/postgresql"
+)
+
+var _ database.Migrator = &Migrator{}
+
+// Migrator represents the database migrator that should be used to migrate from v4 of the database to v5
+type Migrator struct {
+	SQL *sqlx.DB
+}
+
+func NewMigrator(db *postgresql.Database) *Migrator {
+	return &Migrator{
+		SQL: db.SQL,
+	}
+}

database/migrate/v5/types.go

@@ -0,0 +1,10 @@
+package v5
+
+type MessageRow struct {
+	TransactionHash           string `db:"transaction_hash"`
+	Index                     int64  `db:"index"`
+	Type                      string `db:"type"`
+	Value                     string `db:"value"`
+	InvolvedAccountsAddresses string `db:"involved_accounts_addresses"`
+	Height                    int64  `db:"height"`
+}

database/schema/00-cosmos.sql

@@ -66,11 +66,21 @@ CREATE INDEX transaction_hash_index ON transaction (hash);
 CREATE INDEX transaction_height_index ON transaction (height);
 CREATE INDEX transaction_partition_id_index ON transaction (partition_id);
 
+CREATE TABLE message_type
+(
+    type      TEXT   NOT NULL UNIQUE,
+    module    TEXT   NOT NULL,
+    label     TEXT   NOT NULL,
+    height    BIGINT NOT NULL
+);
+CREATE INDEX message_type_module_index ON message_type (module);
+CREATE INDEX message_type_type_index ON message_type (type);
+
 CREATE TABLE message
 (
     transaction_hash            TEXT   NOT NULL,
     index                       BIGINT NOT NULL,
-    type                        TEXT   NOT NULL,
+    type                        TEXT   NOT NULL REFERENCES message_type(type),
     value                       JSONB  NOT NULL,
     involved_accounts_addresses TEXT[] NOT NULL,
 
@@ -101,6 +111,17 @@ WHERE (cardinality(types) = 0 OR type = ANY (types))
 ORDER BY height DESC LIMIT "limit" OFFSET "offset"
 $$ LANGUAGE sql STABLE;
 
+CREATE FUNCTION messages_by_type(
+  types text [],
+  "limit" bigint DEFAULT 100,
+  "offset" bigint DEFAULT 0) 
+  RETURNS SETOF message AS 
+$$ 
+SELECT * FROM message
+WHERE (cardinality(types) = 0 OR type = ANY (types))
+ORDER BY height DESC LIMIT "limit" OFFSET "offset" 
+$$ LANGUAGE sql STABLE;
+
 CREATE TABLE pruning
 (
     last_pruned_height BIGINT NOT NULL

go.mod

@@ -1,6 +1,6 @@
 module github.com/forbole/bdjuno/v4
 
-go 1.19
+go 1.20
 
 require (
 	cosmossdk.io/math v1.2.0
@@ -225,7 +225,7 @@ require (
 	github.com/nunnatsa/ginkgolinter v0.14.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761 // indirect
+	github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polyfloyd/go-errorlint v1.4.5 // indirect

go.sum

@@ -1257,8 +1257,8 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
-github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761 h1:W04oB3d0J01W5jgYRGKsV8LCM6g9EkCvPkZcmFuy0OE=
-github.com/petermattis/goid v0.0.0-20230518223814-80aa455d8761/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
+github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU=
+github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

modules/actions/module.go

@@ -1,12 +1,12 @@
 package actions
 
 import (
-	"github.com/forbole/juno/v5/types/params"
 	"github.com/forbole/juno/v5/modules"
 	"github.com/forbole/juno/v5/node"
 	"github.com/forbole/juno/v5/node/builder"
 	nodeconfig "github.com/forbole/juno/v5/node/config"
 	"github.com/forbole/juno/v5/types/config"
+	"github.com/forbole/juno/v5/types/params"
 
 	modulestypes "github.com/forbole/bdjuno/v4/modules/types"
 )

modules/message_type/handle_msg.go

@@ -0,0 +1,27 @@
+package message_type
+
+import (
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/gogoproto/proto"
+	utils "github.com/forbole/bdjuno/v4/modules/utils"
+	msgtypes "github.com/forbole/bdjuno/v4/types"
+
+	"github.com/forbole/juno/v5/types"
+)
+
+// HandleMsg represents a message handler that stores the given message inside the proper database table
+func (m *Module) HandleMsg(
+	index int, msg sdk.Msg, tx *types.Tx) error {
+	// Save message type
+	err := m.db.SaveMessageType(msgtypes.NewMessageType(
+		proto.MessageName(msg),
+		utils.GetModuleNameFromTypeURL(proto.MessageName(msg)),
+		utils.GetMsgFromTypeURL(proto.MessageName(msg)),
+		tx.Height))
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

modules/message_type/module.go

@@ -0,0 +1,32 @@
+package message_type
+
+import (
+	"github.com/cosmos/cosmos-sdk/codec"
+	"github.com/forbole/bdjuno/v4/database"
+	"github.com/forbole/juno/v5/modules"
+	"github.com/forbole/juno/v5/modules/messages"
+)
+
+var (
+	_ modules.Module        = &Module{}
+	_ modules.MessageModule = &Module{}
+)
+
+type Module struct {
+	cdc    codec.Codec
+	db     *database.Db
+	parser messages.MessageAddressesParser
+}
+
+func NewModule(parser messages.MessageAddressesParser, cdc codec.Codec, db *database.Db) *Module {
+	return &Module{
+		parser: parser,
+		cdc:    cdc,
+		db:     db,
+	}
+}
+
+// Name implements modules.Module
+func (m *Module) Name() string {
+	return "message_type"
+}

modules/registrar.go

@@ -28,11 +28,13 @@ import (
 
 	dailyrefetch "github.com/forbole/bdjuno/v4/modules/daily_refetch"
 	"github.com/forbole/bdjuno/v4/modules/gov"
+	messagetype "github.com/forbole/bdjuno/v4/modules/message_type"
 	"github.com/forbole/bdjuno/v4/modules/mint"
 	"github.com/forbole/bdjuno/v4/modules/modules"
 	"github.com/forbole/bdjuno/v4/modules/pricefeed"
 	"github.com/forbole/bdjuno/v4/modules/staking"
 	"github.com/forbole/bdjuno/v4/modules/upgrade"
+	juno "github.com/forbole/juno/v5/types"
 )
 
 // UniqueAddressesParser returns a wrapper around the given parser that removes all duplicated addresses
@@ -85,6 +87,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 	distrModule := distribution.NewModule(sources.DistrSource, cdc, db)
 
 	feegrantModule := feegrant.NewModule(cdc, db)
+	messagetypeModule := messagetype.NewModule(r.parser, cdc, db)
 	mintModule := mint.NewModule(sources.MintSource, cdc, db)
 
 	slashingModule := slashing.NewModule(sources.SlashingSource, cdc, db)
@@ -109,6 +112,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 		feegrantModule,
 		govModule,
 		mintModule,
+		messagetypeModule,
 		modules.NewModule(ctx.JunoConfig.Chain, db),
 		pricefeed.NewModule(ctx.JunoConfig, cdc, db),
 		slashingModule,

modules/types/sources.go

@@ -12,6 +12,7 @@ import (
 	providersource "github.com/forbole/bdjuno/v4/modules/ccv/provider/source"
 	remoteprovidersource "github.com/forbole/bdjuno/v4/modules/ccv/provider/source/remote"
 	"github.com/forbole/juno/v5/node/remote"
+	"github.com/forbole/juno/v5/types/params"
 
 	banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
 	distributionkeeper "github.com/cosmos/cosmos-sdk/x/distribution/keeper"