-
Notifications
You must be signed in to change notification settings - Fork 90
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
WIP: Batching #1570
WIP: Batching #1570
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
package postgres | ||
|
||
import ( | ||
"fmt" | ||
"sync" | ||
|
||
"github.com/jackc/pgx/v4" | ||
"github.com/jackc/pgx/v4/pgxpool" | ||
log "github.com/sirupsen/logrus" | ||
|
||
"github.com/algorand/indexer/v3/idb/migration" | ||
) | ||
|
||
// const useExperimentalTxnInsertion = false | ||
// const useExperimentalWithIntraBugfix = true | ||
|
||
var serializable = pgx.TxOptions{IsoLevel: pgx.Serializable} // be a real ACID database | ||
var readonlyRepeatableRead = pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly} | ||
|
||
// in actuality, for postgres the following is no weaker than ReadCommitted: | ||
// https://www.postgresql.org/docs/current/transaction-iso.html | ||
// TODO: change this to pgs.ReadCommitted | ||
// var uncommitted = pgx.TxOptions{IsoLevel: pgx.ReadUncommitted} | ||
|
||
|
||
|
||
// var experimentalCommitLevel = uncommitted // serializable // uncommitted | ||
|
||
// IndexerDb is an idb.IndexerDB implementation | ||
type IndexerDb struct { | ||
readonly bool | ||
log *log.Logger | ||
|
||
db *pgxpool.Pool | ||
migration *migration.Migration | ||
accountingLock sync.Mutex | ||
|
||
TuningParams | ||
} | ||
|
||
// TuningParams are database interaction settings that can be | ||
// fine tuned to improve performance based on a specific hardware deployment | ||
// and workload characteristics. | ||
type TuningParams struct { | ||
tzaffi marked this conversation as resolved.
Show resolved
Hide resolved
|
||
PgxOpts pgx.TxOptions | ||
BatchSize uint32 | ||
} | ||
|
||
// defaultTuningParams returns a TuningParams object with default values. | ||
func defaultTuningParams() TuningParams{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🚫 [Lint Errors] reported by reviewdog 🐶 |
||
return TuningParams{ | ||
PgxOpts: serializable, | ||
BatchSize: 2500, | ||
} | ||
} | ||
|
||
func shortName(isoLevel pgx.TxIsoLevel) string { | ||
switch isoLevel { | ||
case pgx.Serializable: | ||
return "S" | ||
case pgx.RepeatableRead: | ||
return "RR" | ||
case pgx.ReadCommitted: | ||
return "RC" | ||
case pgx.ReadUncommitted: | ||
return "RU" | ||
default: | ||
return fmt.Sprintf("unknown_%s", isoLevel) | ||
} | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🚫 [Lint Errors] reported by reviewdog 🐶 |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,13 @@ | |
|
||
var testpg string | ||
|
||
var knownPgImages = map[string]struct{}{ | ||
"13-alpine": {}, | ||
"14": {}, | ||
"15": {}, | ||
"16beta3": {}, | ||
} | ||
|
||
func init() { | ||
flag.StringVar(&testpg, "test-pg", "", "postgres connection string; resets the database") | ||
if testpg == "" { | ||
|
@@ -28,7 +35,7 @@ | |
|
||
// SetupPostgres starts a gnomock postgres DB then returns the database object, | ||
// the connection string and a shutdown function. | ||
func SetupPostgres(t *testing.T) (*pgxpool.Pool, string, func()) { | ||
func SetupPostgres(t testing.TB) (*pgxpool.Pool, string, func()) { | ||
if testpg != "" { | ||
// use non-docker Postgresql | ||
connStr := testpg | ||
|
@@ -47,8 +54,16 @@ | |
return db, connStr, shutdownFunc | ||
} | ||
|
||
return SetupGnomockPgWithVersion(t, "13-alpine") | ||
} | ||
|
||
func SetupGnomockPgWithVersion(t testing.TB, pgImage string) (*pgxpool.Pool, string, func()) { | ||
Check failure on line 60 in idb/postgres/internal/testing/testing.go GitHub Actions / reviewdog-errors
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🚫 [Lint Errors] reported by reviewdog 🐶 |
||
if _, ok := knownPgImages[pgImage]; !ok { | ||
t.Fatalf("SetupGnomockPgWithVersion(): unrecognized postgres Docker image for gnomock preset: %s", pgImage) | ||
} | ||
|
||
p := postgres.Preset( | ||
postgres.WithVersion("13-alpine"), | ||
postgres.WithVersion(pgImage), | ||
postgres.WithUser("gnomock", "gnomick"), | ||
postgres.WithDatabase("mydb"), | ||
) | ||
|
@@ -61,6 +76,16 @@ | |
"gnomock", "gnomick", "mydb", | ||
) | ||
|
||
// config, err := pgxpool.ParseConfig(connStr) | ||
// require.NoError(t, err, "Error parsing connection string: %s", connStr) | ||
|
||
// if maxConns != nil { | ||
// config.MaxConns = *maxConns | ||
// } | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🚫 [Lint Errors] reported by reviewdog 🐶 |
||
// db, err := pgxpool.ConnectConfig(context.Background(), config) | ||
// require.NoError(t, err, "Error creating connection pool via config: %#v", config) | ||
|
||
db, err := pgxpool.Connect(context.Background(), connStr) | ||
require.NoError(t, err, "Error opening postgres connection") | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🚫 [Lint Errors] reported by reviewdog 🐶
File is not
gci
-ed with --skip-generated -s standard -s default -s prefix(github.com/algorand) -s prefix(github.com/algorand/go-algorand) (gci)