support custom key types and key definition registry

There is now a type core/crypto.KeyDefinition that is registered with
core/crypto.RegisterKeyType. An arbitrary external key type can be
supported with basic capabilities defined, including unmarshal/generate/type.
The main purpose is to recognize key types in various config, DBs, and
network comms, and provide the ability to unmarshal and create these keys.

type KeyDefinition interface {
	Type() KeyType      // name of key type e.g. "secp256k1"
	EncodeFlag() uint32 // prefix for compact unique binary encoding

	UnmarshalPrivateKey(b []byte) (PrivateKey, error)
	UnmarshalPublicKey(b []byte) (PublicKey, error)

	Generate() PrivateKey
}

The package level core/crypto.UnmarshalPrivateKey (and one for public)
dispatch to the registered implementation based on KeyType. Note that the
interfaces PrivateKey and PublicKey include the crypto.Key interface, which
has its own marshal method (Bytes()).

KeyType is a string now to make it self describing since we can't really
have methods like Valid and String anymore now that the key type
registry is not in core.

Regarding the weird new EncodeFlag, this is for node internal use to make
compact serializations. See core/crypto.WireEncodeKey etc. It seemed
silly to encode a string key type in non-user-facing places like storage,
network transmission, hashing, etc. So this does create a second value
that must be unique to register, but I don't think it's a problem.

permit eth address in account balance,transfer commands

eth addr in account balance output

app/key/cmd.go

@@ -27,17 +27,27 @@ func KeyCmd() *cobra.Command {
 }
 
 func privKeyInfo(priv crypto.PrivateKey) *PrivateKeyInfo {
+	var keyText, keyFmt string
+	s, ok := priv.(interface{ ASCII() (string, string) })
+	if ok {
+		keyFmt, keyText = s.ASCII()
+	} else {
+		keyText = hex.EncodeToString(priv.Bytes())
+		keyFmt = "hex"
+	}
 	return &PrivateKeyInfo{
-		KeyType:       priv.Type().String(),
-		PrivateKeyHex: hex.EncodeToString(priv.Bytes()),
-		PublicKeyHex:  hex.EncodeToString(priv.Public().Bytes()),
+		KeyType:        priv.Type().String(),
+		PrivateKeyText: keyText,
+		privKeyFmt:     keyFmt,
+		PublicKeyHex:   hex.EncodeToString(priv.Public().Bytes()),
 	}
 }
 
 type PrivateKeyInfo struct {
-	KeyType       string `json:"key_type"`
-	PrivateKeyHex string `json:"private_key_hex"`
-	PublicKeyHex  string `json:"public_key_hex"`
+	KeyType        string `json:"key_type"`
+	PrivateKeyText string `json:"private_key_text"`
+	privKeyFmt     string `json:"-"`
+	PublicKeyHex   string `json:"public_key_hex"`
 }
 
 func (p *PrivateKeyInfo) MarshalJSON() ([]byte, error) {
@@ -47,10 +57,11 @@ func (p *PrivateKeyInfo) MarshalJSON() ([]byte, error) {
 
 func (p *PrivateKeyInfo) MarshalText() ([]byte, error) {
 	return []byte(fmt.Sprintf(`Key type: %s
-Private key (hex): %s
+Private key (%s): %s
 Public key (plain hex): %v`,
 		p.KeyType,
-		p.PrivateKeyHex,
+		p.privKeyFmt,
+		p.PrivateKeyText,
 		p.PublicKeyHex,
 	)), nil
 }

app/key/gen.go

@@ -3,7 +3,6 @@ package key
 import (
 	"encoding/hex"
 	"fmt"
-	"strconv"
 
 	"github.com/spf13/cobra"
 
@@ -35,11 +34,7 @@ func GenCmd() *cobra.Command {
 				var err error
 				keyType, err = crypto.ParseKeyType(args[0])
 				if err != nil {
-					keyTypeInt, err := strconv.ParseUint(args[0], 10, 16)
-					if err != nil {
-						return display.PrintErr(cmd, fmt.Errorf("invalid key type (%s): %w", args[0], err))
-					}
-					keyType = crypto.KeyType(keyTypeInt)
+					return display.PrintErr(cmd, fmt.Errorf("invalid key type (%s): %w", args[0], err))
 				}
 			}
 
@@ -52,11 +47,7 @@ func GenCmd() *cobra.Command {
 				if raw {
 					return display.PrintCmd(cmd, display.RespString(hex.EncodeToString(privKey.Bytes())))
 				}
-				return display.PrintCmd(cmd, &PrivateKeyInfo{
-					KeyType:       keyType.String(),
-					PrivateKeyHex: hex.EncodeToString(privKey.Bytes()),
-					PublicKeyHex:  hex.EncodeToString(privKey.Public().Bytes()),
-				})
+				return display.PrintCmd(cmd, privKeyInfo(privKey))
 			}
 
 			if err := SaveNodeKey(out, privKey); err != nil {

app/key/info.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/kwilteam/kwil-db/app/shared/display"
@@ -45,14 +44,9 @@ func InfoCmd() *cobra.Command {
 				}
 				keyType := crypto.KeyTypeSecp256k1 // default
 				if keyTypeStr != "" {
-					keyTypeInt, err := strconv.ParseUint(keyTypeStr, 10, 16)
-					if err != nil { // maybe it's a string like "secp256k1"
-						keyType, err = crypto.ParseKeyType(keyTypeStr)
-						if err != nil {
-							return display.PrintErr(cmd, fmt.Errorf("invalid key type (%s): %w", keyTypeStr, err))
-						}
-					} else {
-						keyType = crypto.KeyType(keyTypeInt)
+					keyType, err = crypto.ParseKeyType(keyTypeStr)
+					if err != nil {
+						return display.PrintErr(cmd, fmt.Errorf("invalid key type (%s): %w", keyTypeStr, err))
 					}
 				}
 				priv, err := crypto.UnmarshalPrivateKey(keyBts, keyType)
@@ -65,11 +59,7 @@ func InfoCmd() *cobra.Command {
 				if err != nil {
 					return display.PrintErr(cmd, err)
 				}
-				return display.PrintCmd(cmd, &PrivateKeyInfo{
-					KeyType:       key.Type().String(),
-					PrivateKeyHex: hex.EncodeToString(key.Bytes()),
-					PublicKeyHex:  hex.EncodeToString(key.Public().Bytes()),
-				})
+				return display.PrintCmd(cmd, privKeyInfo(key))
 			}
 
 			cmd.Usage()

app/node/node.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/log"
 	"github.com/kwilteam/kwil-db/core/types"
+	authExt "github.com/kwilteam/kwil-db/extensions/auth"
 	"github.com/kwilteam/kwil-db/node"
 	"github.com/kwilteam/kwil-db/node/consensus"
 	"github.com/kwilteam/kwil-db/node/listeners"
@@ -319,7 +320,7 @@ func loadGenesisAndPrivateKey(rootDir string, autogen bool, dbOwner string) (pri
 		genCfg.DBOwner = dbOwner
 	} else {
 		signer := auth.GetUserSigner(privKey)
-		ident, err := auth.GetIdentifierFromSigner(signer)
+		ident, err := authExt.GetIdentifierFromSigner(signer)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to get identifier from user signer: %w", err)
 		}

app/setup/genesis.go

@@ -9,12 +9,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/spf13/cobra"
+
 	"github.com/kwilteam/kwil-db/app/shared/display"
 	"github.com/kwilteam/kwil-db/config"
 	"github.com/kwilteam/kwil-db/core/crypto"
 	"github.com/kwilteam/kwil-db/core/types"
 	"github.com/kwilteam/kwil-db/node"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -76,7 +77,7 @@ func GenesisCmd() *cobra.Command {
 			conf := config.DefaultGenesisConfig()
 			conf, err = mergeGenesisFlags(conf, cmd, &flagCfg)
 			if err != nil {
-				return display.PrintErr(cmd, err)
+				return display.PrintErr(cmd, fmt.Errorf("failed to create genesis file: %w", err))
 			}
 
 			existingFile, err := os.Stat(genesisFile)
@@ -117,9 +118,6 @@ func bindGenesisFlags(cmd *cobra.Command, cfg *genesisFlagConfig) {
 
 // mergeGenesisFlags merges the genesis configuration flags with the given configuration.
 func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *genesisFlagConfig) (*config.GenesisConfig, error) {
-	makeErr := func(e error) error {
-		return display.PrintErr(cmd, fmt.Errorf("failed to create genesis file: %w", e))
-	}
 	if cmd.Flags().Changed("chain-id") {
 		conf.ChainID = flagCfg.chainID
 	}
@@ -129,30 +127,29 @@ func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *
 		for _, v := range flagCfg.validators {
 			parts := strings.Split(v, ":")
 			if len(parts) != 2 {
-				return nil, makeErr(fmt.Errorf("invalid format for validator, expected key:power, received: %s", v))
+				return nil, fmt.Errorf("invalid format for validator, expected key:power, received: %s", v)
 			}
 
 			keyParts := strings.Split(parts[0], "#")
 			hexPub, err := hex.DecodeString(keyParts[0])
 			if err != nil {
-				return nil, makeErr(fmt.Errorf("invalid public key for validator: %s", parts[0]))
+				return nil, fmt.Errorf("invalid public key for validator: %s", parts[0])
 			}
 
 			power, err := strconv.ParseInt(parts[1], 10, 64)
 			if err != nil {
-				return nil, makeErr(fmt.Errorf("invalid power for validator: %s", parts[1]))
+				return nil, fmt.Errorf("invalid power for validator: %s", parts[1])
 			}
 
-			// string to Int
-			keyType, err := strconv.ParseInt(keyParts[1], 10, 64)
+			keyType, err := crypto.ParseKeyType(keyParts[1])
 			if err != nil {
-				return nil, makeErr(fmt.Errorf("invalid power for validator: %s", keyParts[1]))
+				return nil, fmt.Errorf("invalid key type for validator: %s", keyParts[1])
 			}
 
 			conf.Validators = append(conf.Validators, &types.Validator{
 				AccountID: types.AccountID{
 					Identifier: hexPub,
-					KeyType:    crypto.KeyType(keyType),
+					KeyType:    keyType,
 				},
 				Power: power,
 			})
@@ -164,32 +161,27 @@ func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *
 		for _, a := range flagCfg.allocs {
 			parts := strings.Split(a, ":")
 			if len(parts) != 2 {
-				return nil, makeErr(fmt.Errorf("invalid format for alloc, expected address:balance, received: %s", a))
+				return nil, fmt.Errorf("invalid format for alloc, expected id#keyType:balance, received: %s", a)
 			}
 
 			keyParts := strings.Split(parts[0], "#")
 			if len(keyParts) != 2 {
-				return nil, makeErr(fmt.Errorf("invalid address for alloc: %s", parts[0]))
+				return nil, fmt.Errorf("invalid address for alloc: %s", parts[0])
 			}
 
-			keyTypeInt, err := strconv.ParseInt(keyParts[1], 10, 64)
+			keyType, err := crypto.ParseKeyType(keyParts[1])
 			if err != nil {
-				return nil, makeErr(fmt.Errorf("invalid key type for genesis allocs: %s", keyParts[1]))
-			}
-			keyType := crypto.KeyType(keyTypeInt)
-
-			if !keyType.Valid() {
-				return nil, makeErr(fmt.Errorf("invalid key type for genesis allocs: %s", keyParts[1]))
+				return nil, fmt.Errorf("invalid key type for validator: %s", keyParts[1])
 			}
 
 			balance, ok := new(big.Int).SetString(parts[1], 10)
 			if !ok {
-				return nil, makeErr(fmt.Errorf("invalid balance for alloc: %s", parts[1]))
+				return nil, fmt.Errorf("invalid balance for alloc: %s", parts[1])
 			}
 
 			keyStr, err := hex.DecodeString(keyParts[0])
 			if err != nil {
-				return nil, makeErr(fmt.Errorf("invalid address for alloc: %s", keyParts[0]))
+				return nil, fmt.Errorf("invalid address for alloc: %s", keyParts[0])
 			}
 
 			conf.Allocs = append(conf.Allocs, config.GenesisAlloc{
@@ -209,11 +201,11 @@ func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *
 	if cmd.Flags().Changed("leader") {
 		pubkeyBts, keyType, err := config.DecodePubKeyAndType(flagCfg.leader)
 		if err != nil {
-			return nil, makeErr(err)
+			return nil, err
 		}
 		pubkey, err := crypto.UnmarshalPublicKey(pubkeyBts, keyType)
 		if err != nil {
-			return nil, makeErr(err)
+			return nil, err
 		}
 		conf.Leader = types.PublicKey{PublicKey: pubkey}
 	}
@@ -237,7 +229,7 @@ func mergeGenesisFlags(conf *config.GenesisConfig, cmd *cobra.Command, flagCfg *
 	if cmd.Flags().Changed("genesis-state") {
 		hash, err := appHashFromSnapshotFile(flagCfg.genesisState)
 		if err != nil {
-			return nil, makeErr(err)
+			return nil, err
 		}
 		conf.StateHash = hash
 	}

app/setup/init.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/spf13/cobra"
+
 	"github.com/kwilteam/kwil-db/app/custom"
 	"github.com/kwilteam/kwil-db/app/key"
 	"github.com/kwilteam/kwil-db/app/node/conf"
@@ -17,8 +19,8 @@ import (
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/types"
 	"github.com/kwilteam/kwil-db/core/utils"
+	authExt "github.com/kwilteam/kwil-db/extensions/auth"
 	"github.com/kwilteam/kwil-db/node"
-	"github.com/spf13/cobra"
 )
 
 const (
@@ -126,7 +128,7 @@ func InitCmd() *cobra.Command {
 				genCfg = config.DefaultGenesisConfig()
 				genCfg, err = mergeGenesisFlags(genCfg, cmd, &genFlags)
 				if err != nil {
-					return display.PrintErr(cmd, err)
+					return display.PrintErr(cmd, fmt.Errorf("failed to create genesis file: %w", err))
 				}
 
 				genCfg.Leader = types.PublicKey{PublicKey: privKey.Public()}
@@ -141,7 +143,7 @@ func InitCmd() *cobra.Command {
 				// If DB owner is not set, set it to the node's public key
 				if genCfg.DBOwner == "" {
 					signer := auth.GetUserSigner(privKey)
-					ident, err := auth.GetIdentifierFromSigner(signer)
+					ident, err := authExt.GetIdentifierFromSigner(signer)
 					if err != nil {
 						return display.PrintErr(cmd, fmt.Errorf("failed to get identifier for dbOwner: %w", err))
 					}

app/setup/testnet.go

@@ -20,6 +20,7 @@ import (
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
 	"github.com/kwilteam/kwil-db/core/types"
 	ktypes "github.com/kwilteam/kwil-db/core/types"
+	authExt "github.com/kwilteam/kwil-db/extensions/auth"
 	"github.com/kwilteam/kwil-db/node"
 )
 
@@ -157,7 +158,7 @@ func GenerateTestnetConfigs(cfg *TestnetConfig, opts *ConfigOpts) error {
 	genConfig.DBOwner = cfg.Owner
 	if genConfig.DBOwner == "" {
 		signer := auth.GetUserSigner(keys[0])
-		ident, err := auth.GetIdentifierFromSigner(signer)
+		ident, err := authExt.GetIdentifierFromSigner(signer)
 		if err != nil {
 			return fmt.Errorf("failed to get identifier from user signer for dbOwner: %w", err)
 		}

app/snapshot/create.go

@@ -6,7 +6,6 @@ import (
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
-	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -18,6 +17,8 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/spf13/cobra"
+
 	"github.com/kwilteam/kwil-db/app/shared/bind"
 	"github.com/kwilteam/kwil-db/app/shared/display"
 	"github.com/kwilteam/kwil-db/config"
@@ -26,7 +27,6 @@ import (
 	"github.com/kwilteam/kwil-db/node"
 	"github.com/kwilteam/kwil-db/node/meta"
 	"github.com/kwilteam/kwil-db/node/pg"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -234,7 +234,7 @@ func PGDump(ctx context.Context, dbName, dbUser, dbPass, dbHost, dbPort string,
 			}
 
 			// voterID is the encoded public key
-			pubkey, keyType, err := decodePubKey(voterID)
+			pubkey, err := crypto.WireDecodePubKey(voterID)
 			if err != nil {
 				return -1, nil, nil, fmt.Errorf("failed to decode public key: %w", err)
 			}
@@ -247,8 +247,8 @@ func PGDump(ctx context.Context, dbName, dbUser, dbPass, dbHost, dbPort string,
 			// TODO: update this once the keytype is added to the voters table
 			genCfg.Validators = append(genCfg.Validators, &types.Validator{
 				AccountID: types.AccountID{
-					Identifier: pubkey,
-					KeyType:    keyType,
+					Identifier: pubkey.Bytes(),
+					KeyType:    pubkey.Type(),
 				},
 				Power: power,
 			})
@@ -337,12 +337,3 @@ func chainHeight(ctx context.Context, dbName, dbUser, dbPass, dbHost, dbPort str
 
 	return height, nil
 }
-
-func decodePubKey(b []byte) ([]byte, crypto.KeyType, error) {
-	if len(b) <= 4 {
-		return nil, 0, errors.New("insufficient data for public key")
-	}
-
-	keyType := crypto.KeyType(binary.LittleEndian.Uint32(b))
-	return b[4:], keyType, nil
-}