Merge pull request #93 from hyperledger/psql-migration

Migration command - leveldb2postgres

cmd/migrate.go

@@ -0,0 +1,52 @@
+// Copyright © 2023 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+
+	"github.com/hyperledger/firefly-transaction-manager/internal/persistence/dbmigration"
+	"github.com/spf13/cobra"
+)
+
+func MigrateCommand(initConfig func() error) *cobra.Command {
+	return buildMigrateCommand(initConfig)
+}
+
+func buildMigrateCommand(initConfig func() error) *cobra.Command {
+	migrateCmd := &cobra.Command{
+		Use:   "migrate <subcommand>",
+		Short: "Migration tools",
+	}
+	migrateCmd.AddCommand(buildLeveldb2postgresCommand(initConfig))
+
+	return migrateCmd
+}
+
+func buildLeveldb2postgresCommand(initConfig func() error) *cobra.Command {
+	leveldb2postgresEventStreamsCmd := &cobra.Command{
+		Use:   "leveldb2postgres",
+		Short: "Migrate from LevelDB to PostgreSQL persistence",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := initConfig(); err != nil {
+				return err
+			}
+			return dbmigration.MigrateLevelDBToPostgres(context.Background())
+		},
+	}
+	return leveldb2postgresEventStreamsCmd
+}

cmd/migrate_test.go

@@ -0,0 +1,44 @@
+// Copyright © 2023 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hyperledger/firefly-transaction-manager/internal/tmconfig"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrateCommandFailInit(t *testing.T) {
+	cmd := MigrateCommand(func() error {
+		return fmt.Errorf("pop")
+	})
+	cmd.SetArgs([]string{"leveldb2postgres"})
+	err := cmd.Execute()
+	assert.Regexp(t, "pop", err)
+}
+
+func TestMigrateCommandFailRun(t *testing.T) {
+	cmd := MigrateCommand(func() error {
+		tmconfig.Reset()
+		return nil
+	})
+	cmd.SetArgs([]string{"leveldb2postgres"})
+	err := cmd.Execute()
+	assert.Regexp(t, "FF21050", err)
+}

internal/persistence/dbmigration/leveldb2postgres.go

@@ -0,0 +1,46 @@
+// Copyright © 2023 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dbmigration
+
+import (
+	"context"
+	"time"
+
+	"github.com/hyperledger/firefly-common/pkg/i18n"
+	"github.com/hyperledger/firefly-transaction-manager/internal/persistence/leveldb"
+	"github.com/hyperledger/firefly-transaction-manager/internal/persistence/postgres"
+	"github.com/hyperledger/firefly-transaction-manager/internal/tmconfig"
+	"github.com/hyperledger/firefly-transaction-manager/internal/tmmsgs"
+)
+
+func MigrateLevelDBToPostgres(ctx context.Context) (err error) {
+	m := &dbMigration{}
+
+	tmconfig.PostgresSection.Set(postgres.ConfigTXWriterBatchTimeout, 0) // single go-routine, no point in batching
+	tmconfig.PostgresSection.Set(postgres.ConfigTXWriterCount, 1)
+	nonceStateTimeout := 0 * time.Second
+	if m.source, err = leveldb.NewLevelDBPersistence(ctx, nonceStateTimeout); err != nil {
+		return i18n.NewError(ctx, tmmsgs.MsgPersistenceInitFail, "leveldb", err)
+	}
+	defer m.source.Close(ctx)
+	if m.target, err = postgres.NewPostgresPersistence(ctx, tmconfig.PostgresSection, nonceStateTimeout); err != nil {
+		return i18n.NewError(ctx, tmmsgs.MsgPersistenceInitFail, "postgres", err)
+	}
+	defer m.target.Close(ctx)
+
+	return m.run(ctx)
+}

internal/persistence/dbmigration/leveldb2postgres_test.go

@@ -0,0 +1,124 @@
+// Copyright © 2023 Kaleido, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dbmigration
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/hyperledger/firefly-common/pkg/config"
+	"github.com/hyperledger/firefly-common/pkg/dbsql"
+	"github.com/hyperledger/firefly-common/pkg/fftypes"
+	"github.com/hyperledger/firefly-transaction-manager/internal/persistence/postgres"
+	"github.com/hyperledger/firefly-transaction-manager/internal/tmconfig"
+	"github.com/stretchr/testify/assert"
+)
+
+func initPSQL(t *testing.T) (context.Context, func()) {
+	tmconfig.Reset()
+
+	// Configure a test postgres
+	ctx, cancelCtx := context.WithCancel(context.Background())
+
+	dbURL := func(dbname string) string {
+		hostname := os.Getenv("POSTGRES_HOSTNAME")
+		port := os.Getenv("POSTGRES_PORT")
+		password := os.Getenv("POSTGRES_PASSWORD")
+		if hostname == "" {
+			hostname = "localhost"
+		}
+		if port == "" {
+			port = "5432"
+		}
+		if password == "" {
+			password = "f1refly"
+		}
+		return fmt.Sprintf("postgres://postgres:%s@%s:%s/%s?sslmode=disable", password, hostname, port, dbname)
+	}
+	utdbName := "ut_" + fftypes.NewUUID().String()
+
+	// First create the database - using the super user
+	adminDB, err := sql.Open("postgres", dbURL("postgres"))
+	assert.NoError(t, err)
+	_, err = adminDB.Exec(fmt.Sprintf(`CREATE DATABASE "%s";`, utdbName))
+	assert.NoError(t, err)
+	err = adminDB.Close()
+	assert.NoError(t, err)
+
+	tmconfig.PostgresSection.Set(dbsql.SQLConfDatasourceURL, dbURL(utdbName))
+	tmconfig.PostgresSection.Set(dbsql.SQLConfMigrationsDirectory, path.Join("..", "..", "db", "migrations", "postgres"))
+
+	psql := &postgres.Postgres{}
+	err = psql.Database.Init(ctx, psql, tmconfig.PostgresSection)
+	assert.NoError(t, err)
+
+	driver, err := psql.GetMigrationDriver(psql.DB())
+	assert.NoError(t, err)
+	m, err := migrate.NewWithDatabaseInstance(
+		"file://../../../db/migrations/postgres",
+		utdbName,
+		driver,
+	)
+	assert.NoError(t, err)
+	err = m.Up()
+	assert.NoError(t, err)
+
+	return ctx, func() {
+		cancelCtx()
+		err := m.Drop()
+		assert.NoError(t, err)
+	}
+}
+
+func TestMigrateLevelDBToPostgres(t *testing.T) {
+	ctx, done := initPSQL(t)
+	defer done()
+
+	// Configure a test LevelDB
+	dir, err := ioutil.TempDir("", "ldb_*")
+	assert.NoError(t, err)
+	config.Set(tmconfig.PersistenceLevelDBPath, dir)
+
+	// Run empty migration and check no errors
+	err = MigrateLevelDBToPostgres(ctx)
+	assert.NoError(t, err)
+
+}
+
+func TestMigrateLevelDBToPostgresFailPSQL(t *testing.T) {
+	tmconfig.Reset()
+
+	// Configure a test LevelDB
+	dir, err := ioutil.TempDir("", "ldb_*")
+	assert.NoError(t, err)
+	config.Set(tmconfig.PersistenceLevelDBPath, dir)
+
+	err = MigrateLevelDBToPostgres(context.Background())
+	assert.Regexp(t, "FF21049", err)
+}
+
+func TestMigrateLevelDBToPostgresFailLevelDB(t *testing.T) {
+	tmconfig.Reset()
+	err := MigrateLevelDBToPostgres(context.Background())
+	assert.Regexp(t, "FF21049", err)
+}