From 4771ba8123f37b78e9b651f6771a1744df1370a9 Mon Sep 17 00:00:00 2001 From: Peter Broadhurst Date: Sun, 2 Jul 2023 17:44:25 -0400 Subject: [PATCH] Turn off the history compaction by default Signed-off-by: Peter Broadhurst --- config.md | 2 +- internal/persistence/postgres/sqlpersistence.go | 2 +- .../persistence/postgres/sqlpersistence_test.go | 5 ++++- internal/persistence/postgres/transaction_writer.go | 10 ++++++---- internal/persistence/postgres/txhistory_test.go | 13 ++++++++++--- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/config.md b/config.md index 1c9f3f35..998d146e 100644 --- a/config.md +++ b/config.md @@ -207,7 +207,7 @@ |batchTimeout|Duration to hold batch open for new transaction operations before flushing to the DB|[`time.Duration`](https://pkg.go.dev/time#Duration)|`10ms` |cacheSlots|Number of transactions to hold cached metadata for to avoid DB read operations to calculate history|`int`|`1000` |count|Number of transactions writing routines to start|`int`|`5` -|historyCompactionInterval|Duration between cleanup activities on the DB for a transaction with a large history|[`time.Duration`](https://pkg.go.dev/time#Duration)|`5m` +|historyCompactionInterval|Duration between cleanup activities on the DB for a transaction with a large history|[`time.Duration`](https://pkg.go.dev/time#Duration)|`0` |historySummaryLimit|Maximum number of action entries to return embedded in the JSON response object when querying a transaction summary|`int`|`50` ## policyengine diff --git a/internal/persistence/postgres/sqlpersistence.go b/internal/persistence/postgres/sqlpersistence.go index 3eba513a..4cbea421 100644 --- a/internal/persistence/postgres/sqlpersistence.go +++ b/internal/persistence/postgres/sqlpersistence.go @@ -62,7 +62,7 @@ func InitConfig(conf config.Section) { conf.AddKnownKey(ConfigTXWriterCacheSlots, 1000) conf.AddKnownKey(ConfigTXWriterHistorySummaryLimit, 50) // returned on TX status - conf.AddKnownKey(ConfigTXWriterHistoryCompactionInterval, "5m") + conf.AddKnownKey(ConfigTXWriterHistoryCompactionInterval, "0" /* disabled by default */) conf.AddKnownKey(ConfigTXWriterCount, 5) conf.AddKnownKey(ConfigTXWriterBatchTimeout, "10ms") conf.AddKnownKey(ConfigTXWriterBatchSize, 100) diff --git a/internal/persistence/postgres/sqlpersistence_test.go b/internal/persistence/postgres/sqlpersistence_test.go index e9cac418..0c1e9f22 100644 --- a/internal/persistence/postgres/sqlpersistence_test.go +++ b/internal/persistence/postgres/sqlpersistence_test.go @@ -28,7 +28,7 @@ import ( "github.com/stretchr/testify/assert" ) -func newMockSQLPersistence(t *testing.T) (context.Context, *sqlPersistence, sqlmock.Sqlmock, func()) { +func newMockSQLPersistence(t *testing.T, init ...func(dbconf config.Section)) (context.Context, *sqlPersistence, sqlmock.Sqlmock, func()) { ctx, cancelCtx := context.WithCancel(context.Background()) db, dbm := dbsql.NewMockProvider().UTInit() @@ -36,6 +36,9 @@ func newMockSQLPersistence(t *testing.T) (context.Context, *sqlPersistence, sqlm config.RootConfigReset() dbconf := config.RootSection("utdb") InitConfig(dbconf) + for _, fn := range init { + fn(dbconf) + } p, err := newSQLPersistence(ctx, &db.Database, dbconf, 1*time.Hour) assert.NoError(t, err) diff --git a/internal/persistence/postgres/transaction_writer.go b/internal/persistence/postgres/transaction_writer.go index d7b29aea..9a5beadf 100644 --- a/internal/persistence/postgres/transaction_writer.go +++ b/internal/persistence/postgres/transaction_writer.go @@ -431,10 +431,12 @@ func (tw *transactionWriter) executeBatchOps(ctx context.Context, b *transaction } } // Do the compression checks - for txID := range b.compressionChecks { - if err := tw.compressionCheck(ctx, txID); err != nil { - log.L(ctx).Errorf("Compression check for %s failed: %s", txID, err) - return err + if tw.compressionInterval > 0 { + for txID := range b.compressionChecks { + if err := tw.compressionCheck(ctx, txID); err != nil { + log.L(ctx).Errorf("Compression check for %s failed: %s", txID, err) + return err + } } } // Do all the transaction deletes diff --git a/internal/persistence/postgres/txhistory_test.go b/internal/persistence/postgres/txhistory_test.go index 584ca819..162f0a0e 100644 --- a/internal/persistence/postgres/txhistory_test.go +++ b/internal/persistence/postgres/txhistory_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/hyperledger/firefly-common/pkg/config" "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly-transaction-manager/internal/persistence" "github.com/hyperledger/firefly-transaction-manager/pkg/apitypes" @@ -35,7 +36,9 @@ func TestTXHistoryCompressionPSQL(t *testing.T) { logrus.SetLevel(logrus.TraceLevel) // Do a set of transaction operations through the writers, and confirm the results are correct - ctx, p, _, done := initTestPSQL(t) + ctx, p, _, done := initTestPSQL(t, func(conf config.Section) { + conf.Set(ConfigTXWriterHistoryCompactionInterval, "5m") + }) defer done() // Write an initial transaction @@ -182,7 +185,9 @@ func TestTXHistoryCompressionPSQL(t *testing.T) { } func TestCompactionFail(t *testing.T) { - ctx, p, mdb, done := newMockSQLPersistence(t) + ctx, p, mdb, done := newMockSQLPersistence(t, func(dbconf config.Section) { + dbconf.Set(ConfigTXWriterHistoryCompactionInterval, "5m") + }) defer done() longAgo := time.Now().Add(-1000 * time.Hour) @@ -238,7 +243,9 @@ func TestCompactionFail(t *testing.T) { } func TestCompactionOnCacheMiss(t *testing.T) { - ctx, p, mdb, done := newMockSQLPersistence(t) + ctx, p, mdb, done := newMockSQLPersistence(t, func(dbconf config.Section) { + dbconf.Set(ConfigTXWriterHistoryCompactionInterval, "5m") + }) defer done() b := &transactionWriterBatch{