From 9732e57a4a5838f2b5d49fad74f37d8b6e12d43e Mon Sep 17 00:00:00 2001 From: tianzhou Date: Thu, 30 Oct 2025 15:45:01 +0800 Subject: [PATCH 1/3] feat: add external database support for plan command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow users to optionally provide an external PostgreSQL database for plan generation instead of always using embedded postgres. This addresses limitations in issues #121 and #122. Key changes: - Add DesiredStateProvider interface to abstract embedded/external DB - Implement ExternalDatabase for user-provided databases - Add --plan-host, --plan-port, --plan-db, --plan-user, --plan-password flags - Support PGSCHEMA_PLAN_* environment variables - Create temporary schemas with nanosecond timestamps for isolation - Validate major version compatibility between plan and target DB - Maintain backwards compatibility (embedded postgres remains default) The external database creates temporary schemas (pgschema_plan_) and cleans them up after plan generation (best effort). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- cmd/apply/apply.go | 28 +-- cmd/plan/external_db_integration_test.go | 206 +++++++++++++++++++++++ cmd/plan/plan.go | 120 +++++++++++-- cmd/util/env.go | 36 ++++ go.mod | 7 +- go.sum | 9 + internal/postgres/desired_state.go | 28 +++ internal/postgres/embedded.go | 9 + internal/postgres/external.go | 162 ++++++++++++++++++ 9 files changed, 572 insertions(+), 33 deletions(-) create mode 100644 cmd/plan/external_db_integration_test.go create mode 100644 internal/postgres/desired_state.go create mode 100644 internal/postgres/external.go diff --git a/cmd/apply/apply.go b/cmd/apply/apply.go index 6ac8915c..b23eaaf4 100644 --- a/cmd/apply/apply.go +++ b/cmd/apply/apply.go @@ -86,11 +86,11 @@ type ApplyConfig struct { // ApplyMigration applies a migration plan to update a database schema. // The caller must provide either: // - A pre-generated plan in config.Plan, OR -// - A desired state file in config.File with a non-nil embeddedPG instance +// - A desired state file in config.File with a non-nil provider instance // -// If config.File is provided, embeddedPG is used to generate the plan. -// The caller is responsible for managing the embeddedPG lifecycle (creation and cleanup). -func ApplyMigration(config *ApplyConfig, embeddedPG *postgres.EmbeddedPostgres) error { +// If config.File is provided, provider is used to generate the plan. +// The caller is responsible for managing the provider lifecycle (creation and cleanup). +func ApplyMigration(config *ApplyConfig, provider postgres.DesiredStateProvider) error { var migrationPlan *plan.Plan var err error @@ -98,9 +98,9 @@ func ApplyMigration(config *ApplyConfig, embeddedPG *postgres.EmbeddedPostgres) if config.Plan != nil { migrationPlan = config.Plan } else if config.File != "" { - // Generate plan from file (requires embeddedPG) - if embeddedPG == nil { - return fmt.Errorf("embeddedPG is required when generating plan from file") + // Generate plan from file (requires provider) + if provider == nil { + return fmt.Errorf("provider is required when generating plan from file") } planConfig := &planCmd.PlanConfig{ @@ -115,7 +115,7 @@ func ApplyMigration(config *ApplyConfig, embeddedPG *postgres.EmbeddedPostgres) } // Generate plan using shared logic - migrationPlan, err = planCmd.GeneratePlan(planConfig, embeddedPG) + migrationPlan, err = planCmd.GeneratePlan(planConfig, provider) if err != nil { return err } @@ -254,7 +254,7 @@ func RunApply(cmd *cobra.Command, args []string) error { ApplicationName: applyApplicationName, } - var embeddedPG *postgres.EmbeddedPostgres + var provider postgres.DesiredStateProvider var err error // If using --plan flag, load plan from JSON file @@ -283,10 +283,10 @@ func RunApply(cmd *cobra.Command, args []string) error { config.Plan = migrationPlan } else { - // Using --file flag, will need embedded postgres + // Using --file flag, will need desired state provider config.File = applyFile - // Create embedded PostgreSQL for desired state validation + // Create desired state provider (embedded postgres or external database) planConfig := &planCmd.PlanConfig{ Host: applyHost, Port: applyPort, @@ -297,15 +297,15 @@ func RunApply(cmd *cobra.Command, args []string) error { File: applyFile, ApplicationName: applyApplicationName, } - embeddedPG, err = planCmd.CreateEmbeddedPostgresForPlan(planConfig) + provider, err = planCmd.CreateDesiredStateProvider(planConfig) if err != nil { return err } - defer embeddedPG.Stop() + defer provider.Stop() } // Apply the migration - return ApplyMigration(config, embeddedPG) + return ApplyMigration(config, provider) } // validateSchemaFingerprint validates that the current database schema matches the expected fingerprint diff --git a/cmd/plan/external_db_integration_test.go b/cmd/plan/external_db_integration_test.go new file mode 100644 index 00000000..195899cd --- /dev/null +++ b/cmd/plan/external_db_integration_test.go @@ -0,0 +1,206 @@ +package plan + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/pgschema/pgschema/internal/postgres" + "github.com/pgschema/pgschema/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestExternalDatabase_BasicFunctionality tests that external database can be used for desired state +func TestExternalDatabase_BasicFunctionality(t *testing.T) { + // Skip in short mode + if testing.Short() { + t.Skip("skipping integration test") + } + + // Setup: Create two embedded postgres instances + // One serves as "target" database, one serves as "external plan" database + targetDB := testutil.SetupPostgres(t) + defer targetDB.Stop() + + externalPlanDB := testutil.SetupPostgres(t) + defer externalPlanDB.Stop() + + // Get connection details + targetHost, targetPort, targetDatabase, targetUser, targetPassword := targetDB.GetConnectionDetails() + planHost, planPort, planDatabase, planUser, planPassword := externalPlanDB.GetConnectionDetails() + + // Create test schema file + schemaSQL := ` +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + email TEXT UNIQUE NOT NULL +); + +CREATE INDEX idx_users_email ON users(email); +` + + tmpDir := t.TempDir() + schemaFile := filepath.Join(tmpDir, "schema.sql") + err := os.WriteFile(schemaFile, []byte(schemaSQL), 0644) + require.NoError(t, err) + + // Create config with external database + config := &PlanConfig{ + Host: targetHost, + Port: targetPort, + DB: targetDatabase, + User: targetUser, + Password: targetPassword, + Schema: "public", + File: schemaFile, + ApplicationName: "pgschema-test", + // External database configuration + PlanDBHost: planHost, + PlanDBPort: planPort, + PlanDBDatabase: planDatabase, + PlanDBUser: planUser, + PlanDBPassword: planPassword, + } + + // Create external database provider + provider, err := CreateDesiredStateProvider(config) + require.NoError(t, err, "should create external database provider") + defer provider.Stop() + + // Verify it's an external database (not embedded) + _, ok := provider.(*postgres.ExternalDatabase) + assert.True(t, ok, "provider should be ExternalDatabase when plan-host is provided") + + // Verify temporary schema name is returned + tempSchema := provider.GetSchemaName() + assert.NotEmpty(t, tempSchema, "temporary schema name should not be empty") + assert.Contains(t, tempSchema, "pgschema_plan_", "temporary schema should have timestamp prefix") + + // Generate plan + migrationPlan, err := GeneratePlan(config, provider) + require.NoError(t, err, "should generate plan") + + // Verify plan has changes (target is empty, desired has tables) + assert.NotNil(t, migrationPlan) + assert.True(t, len(migrationPlan.Groups) > 0, "plan should have at least one group") +} + +// TestExternalDatabase_VersionMismatch tests version compatibility checking +func TestExternalDatabase_VersionMismatch(t *testing.T) { + // Skip in short mode + if testing.Short() { + t.Skip("skipping integration test") + } + + // This test would require two different PostgreSQL versions, which is complex to setup + // For now, we just verify that version detection works + targetDB := testutil.SetupPostgres(t) + defer targetDB.Stop() + + targetHost, targetPort, targetDatabase, targetUser, targetPassword := targetDB.GetConnectionDetails() + + // Detect version from target database + pgVersion, err := postgres.DetectPostgresVersionFromDB( + targetHost, + targetPort, + targetDatabase, + targetUser, + targetPassword, + ) + require.NoError(t, err, "should detect PostgreSQL version") + assert.NotEmpty(t, pgVersion, "version should not be empty") +} + +// TestExternalDatabase_CleanupOnError tests that temporary schema is cleaned up on errors +func TestExternalDatabase_CleanupOnError(t *testing.T) { + // Skip in short mode + if testing.Short() { + t.Skip("skipping integration test") + } + + // Setup external plan database + externalPlanDB := testutil.SetupPostgres(t) + defer externalPlanDB.Stop() + + planHost, planPort, planDatabase, planUser, planPassword := externalPlanDB.GetConnectionDetails() + + // Create external database connection with correct version + externalConfig := &postgres.ExternalDatabaseConfig{ + Host: planHost, + Port: planPort, + Database: planDatabase, + Username: planUser, + Password: planPassword, + TargetMajorVersion: 17, // Assuming test uses PG 17 + } + + extDB, err := postgres.NewExternalDatabase(externalConfig) + require.NoError(t, err) + + tempSchema := extDB.GetSchemaName() + require.NotEmpty(t, tempSchema) + + // Apply some SQL to create the schema + ctx := context.Background() + err = extDB.ApplySchema(ctx, "public", "CREATE TABLE test (id INT);") + require.NoError(t, err) + + // Verify schema exists by checking connection details + host, port, db, user, pass := extDB.GetConnectionDetails() + assert.Equal(t, planHost, host) + assert.Equal(t, planPort, port) + assert.Equal(t, planDatabase, db) + assert.Equal(t, planUser, user) + assert.Equal(t, planPassword, pass) + + // Stop should clean up the temporary schema (best effort) + err = extDB.Stop() + assert.NoError(t, err, "cleanup should not error") +} + +// TestExternalDatabase_SchemaIsolation tests that temporary schemas don't interfere with each other +func TestExternalDatabase_SchemaIsolation(t *testing.T) { + // Skip in short mode + if testing.Short() { + t.Skip("skipping integration test") + } + + // Setup external plan database + externalPlanDB := testutil.SetupPostgres(t) + defer externalPlanDB.Stop() + + planHost, planPort, planDatabase, planUser, planPassword := externalPlanDB.GetConnectionDetails() + + // Create two external database connections + externalConfig1 := &postgres.ExternalDatabaseConfig{ + Host: planHost, + Port: planPort, + Database: planDatabase, + Username: planUser, + Password: planPassword, + TargetMajorVersion: 17, + } + extDB1, err := postgres.NewExternalDatabase(externalConfig1) + require.NoError(t, err) + defer extDB1.Stop() + + externalConfig2 := &postgres.ExternalDatabaseConfig{ + Host: planHost, + Port: planPort, + Database: planDatabase, + Username: planUser, + Password: planPassword, + TargetMajorVersion: 17, + } + extDB2, err := postgres.NewExternalDatabase(externalConfig2) + require.NoError(t, err) + defer extDB2.Stop() + + // Verify different schema names due to timestamp + schema1 := extDB1.GetSchemaName() + schema2 := extDB2.GetSchemaName() + assert.NotEqual(t, schema1, schema2, "temporary schemas should have unique names") +} diff --git a/cmd/plan/plan.go b/cmd/plan/plan.go index ec5daeaf..d85d4d77 100644 --- a/cmd/plan/plan.go +++ b/cmd/plan/plan.go @@ -27,6 +27,13 @@ var ( outputJSON string outputSQL string planNoColor bool + + // Plan database flags (optional - if not provided, uses embedded postgres) + planDBHost string + planDBPort int + planDBDatabase string + planDBUser string + planDBPassword string ) var PlanCmd = &cobra.Command{ @@ -50,6 +57,13 @@ func init() { // Desired state schema file flag PlanCmd.Flags().StringVar(&planFile, "file", "", "Path to desired state SQL schema file (required)") + // Plan database connection flags (optional - for using external database instead of embedded postgres) + PlanCmd.Flags().StringVar(&planDBHost, "plan-host", "", "Plan database host (env: PGSCHEMA_PLAN_HOST). If provided, uses external database instead of embedded postgres") + PlanCmd.Flags().IntVar(&planDBPort, "plan-port", 5432, "Plan database port (env: PGSCHEMA_PLAN_PORT)") + PlanCmd.Flags().StringVar(&planDBDatabase, "plan-db", "", "Plan database name (env: PGSCHEMA_PLAN_DB)") + PlanCmd.Flags().StringVar(&planDBUser, "plan-user", "", "Plan database user (env: PGSCHEMA_PLAN_USER)") + PlanCmd.Flags().StringVar(&planDBPassword, "plan-password", "", "Plan database password (env: PGSCHEMA_PLAN_PASSWORD)") + // Output flags PlanCmd.Flags().StringVar(&outputHuman, "output-human", "", "Output human-readable format to stdout or file path") PlanCmd.Flags().StringVar(&outputJSON, "output-json", "", "Output JSON format to stdout or file path") @@ -60,6 +74,14 @@ func init() { } func runPlan(cmd *cobra.Command, args []string) error { + // Apply environment variables to plan database flags + util.ApplyPlanDBEnvVars(cmd, &planDBHost, &planDBDatabase, &planDBUser, &planDBPassword, &planDBPort) + + // Validate plan database flags if plan-host is provided + if err := util.ValidatePlanDBFlags(planDBHost, planDBDatabase, planDBUser); err != nil { + return err + } + // Derive final password: use provided password or check environment variable finalPassword := planPassword if finalPassword == "" { @@ -68,6 +90,14 @@ func runPlan(cmd *cobra.Command, args []string) error { } } + // Derive final plan database password + finalPlanPassword := planDBPassword + if finalPlanPassword == "" { + if envPassword := os.Getenv("PGSCHEMA_PLAN_PASSWORD"); envPassword != "" { + finalPlanPassword = envPassword + } + } + // Create plan configuration config := &PlanConfig{ Host: planHost, @@ -78,17 +108,23 @@ func runPlan(cmd *cobra.Command, args []string) error { Schema: planSchema, File: planFile, ApplicationName: "pgschema", + // Plan database configuration + PlanDBHost: planDBHost, + PlanDBPort: planDBPort, + PlanDBDatabase: planDBDatabase, + PlanDBUser: planDBUser, + PlanDBPassword: finalPlanPassword, } - // Create embedded PostgreSQL for desired state validation - embeddedPG, err := CreateEmbeddedPostgresForPlan(config) + // Create desired state provider (embedded postgres or external database) + provider, err := CreateDesiredStateProvider(config) if err != nil { return err } - defer embeddedPG.Stop() + defer provider.Stop() // Generate plan - migrationPlan, err := GeneratePlan(config, embeddedPG) + migrationPlan, err := GeneratePlan(config, provider) if err != nil { return err } @@ -119,12 +155,18 @@ type PlanConfig struct { Schema string File string ApplicationName string + // Plan database configuration (optional - for external database) + PlanDBHost string + PlanDBPort int + PlanDBDatabase string + PlanDBUser string + PlanDBPassword string } -// CreateEmbeddedPostgresForPlan creates a temporary embedded PostgreSQL instance -// for validating the desired state schema. The instance should be stopped by the caller. -func CreateEmbeddedPostgresForPlan(config *PlanConfig) (*postgres.EmbeddedPostgres, error) { - // Detect target database PostgreSQL version +// CreateDesiredStateProvider creates either an embedded PostgreSQL instance or connects to an external database +// for validating the desired state schema. The caller is responsible for calling Stop() on the returned provider. +func CreateDesiredStateProvider(config *PlanConfig) (postgres.DesiredStateProvider, error) { + // Detect target database PostgreSQL version (needed for both embedded and external) pgVersion, err := postgres.DetectPostgresVersionFromDB( config.Host, config.Port, @@ -136,6 +178,34 @@ func CreateEmbeddedPostgresForPlan(config *PlanConfig) (*postgres.EmbeddedPostgr return nil, fmt.Errorf("failed to detect PostgreSQL version: %w", err) } + // Extract major version from embedded postgres version string (e.g., "16.9.0" -> 16) + // The version string format is "XX.Y.Z" where XX is the major version + var targetMajorVersion int + _, err = fmt.Sscanf(string(pgVersion), "%d.", &targetMajorVersion) + if err != nil { + return nil, fmt.Errorf("failed to parse PostgreSQL version %s: %w", pgVersion, err) + } + + // If plan-host is provided, use external database + if config.PlanDBHost != "" { + externalConfig := &postgres.ExternalDatabaseConfig{ + Host: config.PlanDBHost, + Port: config.PlanDBPort, + Database: config.PlanDBDatabase, + Username: config.PlanDBUser, + Password: config.PlanDBPassword, + TargetMajorVersion: targetMajorVersion, + } + return postgres.NewExternalDatabase(externalConfig) + } + + // Otherwise, use embedded PostgreSQL + return CreateEmbeddedPostgresForPlan(config, pgVersion) +} + +// CreateEmbeddedPostgresForPlan creates a temporary embedded PostgreSQL instance +// for validating the desired state schema. The instance should be stopped by the caller. +func CreateEmbeddedPostgresForPlan(config *PlanConfig, pgVersion postgres.PostgresVersion) (*postgres.EmbeddedPostgres, error) { // Start embedded PostgreSQL with matching version embeddedConfig := &postgres.EmbeddedPostgresConfig{ Version: pgVersion, @@ -152,9 +222,9 @@ func CreateEmbeddedPostgresForPlan(config *PlanConfig) (*postgres.EmbeddedPostgr } // GeneratePlan generates a migration plan from configuration. -// The caller must provide a non-nil embeddedPG instance for validating the desired state schema. -// The caller is responsible for managing the embeddedPG lifecycle (creation and cleanup). -func GeneratePlan(config *PlanConfig, embeddedPG *postgres.EmbeddedPostgres) (*plan.Plan, error) { +// The caller must provide a non-nil provider instance for validating the desired state schema. +// The caller is responsible for managing the provider lifecycle (creation and cleanup). +func GeneratePlan(config *PlanConfig, provider postgres.DesiredStateProvider) (*plan.Plan, error) { // Load ignore configuration ignoreConfig, err := util.LoadIgnoreFileWithStructure() if err != nil { @@ -182,16 +252,25 @@ func GeneratePlan(config *PlanConfig, embeddedPG *postgres.EmbeddedPostgres) (*p ctx := context.Background() - // Apply desired state SQL to embedded PostgreSQL (resets schema first) - if err := embeddedPG.ApplySchema(ctx, config.Schema, desiredState); err != nil { - return nil, fmt.Errorf("failed to apply desired state to embedded PostgreSQL: %w", err) + // Apply desired state SQL to the provider (embedded postgres or external database) + if err := provider.ApplySchema(ctx, config.Schema, desiredState); err != nil { + return nil, fmt.Errorf("failed to apply desired state: %w", err) + } + + // Inspect the provider database to get desired state IR + providerHost, providerPort, providerDB, providerUsername, providerPassword := provider.GetConnectionDetails() + + // Determine which schema to inspect + // For external database: use the temporary schema name + // For embedded postgres: use the config.Schema (GetSchemaName returns empty string) + schemaToInspect := provider.GetSchemaName() + if schemaToInspect == "" { + schemaToInspect = config.Schema } - // Inspect embedded PostgreSQL to get desired state IR - embeddedHost, embeddedPort, embeddedDB, embeddedUsername, embeddedPassword := embeddedPG.GetConnectionDetails() - desiredStateIR, err := util.GetIRFromDatabase(embeddedHost, embeddedPort, embeddedDB, embeddedUsername, embeddedPassword, config.Schema, config.ApplicationName, ignoreConfig) + desiredStateIR, err := util.GetIRFromDatabase(providerHost, providerPort, providerDB, providerUsername, providerPassword, schemaToInspect, config.ApplicationName, ignoreConfig) if err != nil { - return nil, fmt.Errorf("failed to get desired state from embedded PostgreSQL: %w", err) + return nil, fmt.Errorf("failed to get desired state: %w", err) } // Generate diff (current -> desired) using IR directly @@ -300,4 +379,9 @@ func ResetFlags() { outputJSON = "" outputSQL = "" planNoColor = false + planDBHost = "" + planDBPort = 5432 + planDBDatabase = "" + planDBUser = "" + planDBPassword = "" } diff --git a/cmd/util/env.go b/cmd/util/env.go index 311888c9..32c96c9d 100644 --- a/cmd/util/env.go +++ b/cmd/util/env.go @@ -75,4 +75,40 @@ func PreRunEWithEnvVarsAndConnectionAndApp(dbPtr, userPtr *string, hostPtr *stri return nil } +} + +// ApplyPlanDBEnvVars applies environment variables to plan database connection parameters +// This is used in the plan command to populate plan-* flags from PGSCHEMA_PLAN_* environment variables +func ApplyPlanDBEnvVars(cmd *cobra.Command, hostPtr, dbPtr, userPtr, passwordPtr *string, portPtr *int) { + // Apply environment variables if flags were not explicitly set + if GetEnvWithDefault("PGSCHEMA_PLAN_HOST", "") != "" && !cmd.Flags().Changed("plan-host") { + *hostPtr = GetEnvWithDefault("PGSCHEMA_PLAN_HOST", "") + } + if GetEnvIntWithDefault("PGSCHEMA_PLAN_PORT", 0) != 0 && !cmd.Flags().Changed("plan-port") { + *portPtr = GetEnvIntWithDefault("PGSCHEMA_PLAN_PORT", 0) + } + if GetEnvWithDefault("PGSCHEMA_PLAN_DB", "") != "" && !cmd.Flags().Changed("plan-db") { + *dbPtr = GetEnvWithDefault("PGSCHEMA_PLAN_DB", "") + } + if GetEnvWithDefault("PGSCHEMA_PLAN_USER", "") != "" && !cmd.Flags().Changed("plan-user") { + *userPtr = GetEnvWithDefault("PGSCHEMA_PLAN_USER", "") + } + if GetEnvWithDefault("PGSCHEMA_PLAN_PASSWORD", "") != "" && !cmd.Flags().Changed("plan-password") { + *passwordPtr = GetEnvWithDefault("PGSCHEMA_PLAN_PASSWORD", "") + } +} + +// ValidatePlanDBFlags validates plan database flags when plan-host is provided +// Ensures required flags are present for external database usage +func ValidatePlanDBFlags(planHost, planDB, planUser string) error { + if planHost != "" { + // If plan-host is provided, require plan-db and plan-user + if planDB == "" { + return fmt.Errorf("--plan-db is required when --plan-host is provided (or use PGSCHEMA_PLAN_DB)") + } + if planUser == "" { + return fmt.Errorf("--plan-user is required when --plan-host is provided (or use PGSCHEMA_PLAN_USER)") + } + } + return nil } \ No newline at end of file diff --git a/go.mod b/go.mod index 31a6ebe8..4397a376 100644 --- a/go.mod +++ b/go.mod @@ -12,17 +12,22 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 github.com/spf13/cobra v1.9.1 + github.com/stretchr/testify v1.10.0 golang.org/x/sync v0.17.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.6 // indirect - github.com/stretchr/testify v1.10.0 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect golang.org/x/crypto v0.37.0 // indirect golang.org/x/text v0.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0f9521b6..83c6704f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -20,10 +21,16 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= @@ -45,6 +52,8 @@ golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/postgres/desired_state.go b/internal/postgres/desired_state.go new file mode 100644 index 00000000..5e5dc968 --- /dev/null +++ b/internal/postgres/desired_state.go @@ -0,0 +1,28 @@ +// Package postgres provides PostgreSQL functionality for desired state management. +// This file defines the interface for desired state providers (embedded or external databases). +package postgres + +import "context" + +// DesiredStateProvider is an interface that abstracts the desired state database provider. +// It can be implemented by either embedded PostgreSQL or an external database connection. +type DesiredStateProvider interface { + // GetConnectionDetails returns connection details for IR inspection + // Returns: host, port, database, username, password + GetConnectionDetails() (string, int, string, string, string) + + // GetSchemaName returns the actual schema name to inspect. + // For embedded postgres: returns the user-provided schema name + // For external database: returns the temporary schema name with timestamp + GetSchemaName() string + + // ApplySchema applies the desired state SQL to a schema. + // For embedded postgres: resets the schema (drop/recreate) + // For external database: creates temporary schema with timestamp suffix + ApplySchema(ctx context.Context, schema string, sql string) error + + // Stop performs cleanup. + // For embedded postgres: stops instance and removes temp directory + // For external database: drops temporary schema (best effort) and closes connection + Stop() error +} diff --git a/internal/postgres/embedded.go b/internal/postgres/embedded.go index 1fce0b9d..78f46f55 100644 --- a/internal/postgres/embedded.go +++ b/internal/postgres/embedded.go @@ -170,6 +170,15 @@ func (ep *EmbeddedPostgres) GetConnectionDetails() (host string, port int, datab return ep.host, ep.port, ep.database, ep.username, ep.password } +// GetSchemaName returns the schema name to inspect. +// For embedded postgres, this is managed externally, so we return empty string +// and rely on the caller to track the schema name. +func (ep *EmbeddedPostgres) GetSchemaName() string { + // Embedded postgres doesn't track schema name - it's provided by the caller in ApplySchema + // The caller (GeneratePlan) needs to use config.Schema for inspection + return "" +} + // ApplySchema resets a schema (drops and recreates it) and applies SQL to it. // This ensures a clean state before applying the desired schema definition. func (ep *EmbeddedPostgres) ApplySchema(ctx context.Context, schema string, sql string) error { diff --git a/internal/postgres/external.go b/internal/postgres/external.go new file mode 100644 index 00000000..1d12b18d --- /dev/null +++ b/internal/postgres/external.go @@ -0,0 +1,162 @@ +// Package postgres provides external PostgreSQL database functionality for desired state management. +package postgres + +import ( + "context" + "database/sql" + "fmt" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" +) + +// ExternalDatabase manages an external PostgreSQL database for desired state validation. +// It creates temporary schemas with timestamp suffixes to avoid conflicts. +type ExternalDatabase struct { + db *sql.DB + host string + port int + database string + username string + password string + tempSchema string // Temporary schema name with timestamp suffix + targetMajorVersion int // Expected major version (from target database) +} + +// ExternalDatabaseConfig holds configuration for connecting to an external database +type ExternalDatabaseConfig struct { + Host string + Port int + Database string + Username string + Password string + TargetMajorVersion int // Expected major version to match +} + +// NewExternalDatabase creates a new external database connection for desired state validation. +// It validates the connection, checks version compatibility, and generates a temporary schema name. +func NewExternalDatabase(config *ExternalDatabaseConfig) (*ExternalDatabase, error) { + // Build connection string + dsn := fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=prefer", + config.Username, config.Password, config.Host, config.Port, config.Database) + + // Connect to database + db, err := sql.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("failed to connect to external database: %w", err) + } + + // Test the connection + ctx := context.Background() + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("failed to ping external database: %w", err) + } + + // Detect version and validate compatibility + majorVersion, err := detectMajorVersion(db) + if err != nil { + db.Close() + return nil, fmt.Errorf("failed to detect PostgreSQL version: %w", err) + } + + // Validate version compatibility (require exact major version match) + if majorVersion != config.TargetMajorVersion { + db.Close() + return nil, fmt.Errorf( + "version mismatch: plan database is PostgreSQL %d, but target database is PostgreSQL %d (exact major version match required)", + majorVersion, config.TargetMajorVersion, + ) + } + + // Generate temporary schema name with timestamp including nanoseconds for uniqueness + // Format: pgschema_plan_YYYYMMDD_HHMMSS_NNNNNNNNN + timestamp := time.Now().Format("20060102_150405.000000000") + tempSchema := fmt.Sprintf("pgschema_plan_%s", timestamp) + + return &ExternalDatabase{ + db: db, + host: config.Host, + port: config.Port, + database: config.Database, + username: config.Username, + password: config.Password, + tempSchema: tempSchema, + targetMajorVersion: config.TargetMajorVersion, + }, nil +} + +// GetConnectionDetails returns all connection details needed to connect to the external database +func (ed *ExternalDatabase) GetConnectionDetails() (host string, port int, database, username, password string) { + return ed.host, ed.port, ed.database, ed.username, ed.password +} + +// GetSchemaName returns the temporary schema name used for desired state validation +func (ed *ExternalDatabase) GetSchemaName() string { + return ed.tempSchema +} + +// ApplySchema creates a temporary schema and applies SQL to it. +// The temporary schema name includes a timestamp to avoid conflicts. +func (ed *ExternalDatabase) ApplySchema(ctx context.Context, schema string, sql string) error { + // Note: We use the temporary schema name (ed.tempSchema) instead of the user-provided schema name + // This ensures we don't interfere with existing schemas in the external database + + // Create the temporary schema + createSchemaSQL := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS \"%s\"", ed.tempSchema) + if _, err := ed.db.ExecContext(ctx, createSchemaSQL); err != nil { + return fmt.Errorf("failed to create temporary schema %s: %w", ed.tempSchema, err) + } + + // Set search_path to the temporary schema + setSearchPathSQL := fmt.Sprintf("SET search_path TO \"%s\"", ed.tempSchema) + if _, err := ed.db.ExecContext(ctx, setSearchPathSQL); err != nil { + return fmt.Errorf("failed to set search_path: %w", err) + } + + // Execute the SQL directly + // Note: Desired state SQL should never contain operations like CREATE INDEX CONCURRENTLY + // that cannot run in transactions. Those are migration details, not state declarations. + if _, err := ed.db.ExecContext(ctx, sql); err != nil { + return fmt.Errorf("failed to apply schema SQL to temporary schema %s: %w", ed.tempSchema, err) + } + + return nil +} + +// Stop closes the connection and drops the temporary schema (best effort). +// Errors during cleanup are logged but don't cause failures. +func (ed *ExternalDatabase) Stop() error { + // Drop the temporary schema (best effort - don't fail if this errors) + if ed.db != nil && ed.tempSchema != "" { + ctx := context.Background() + dropSchemaSQL := fmt.Sprintf("DROP SCHEMA IF EXISTS \"%s\" CASCADE", ed.tempSchema) + // Ignore errors - this is best effort cleanup + _, _ = ed.db.ExecContext(ctx, dropSchemaSQL) + } + + // Close database connection + if ed.db != nil { + return ed.db.Close() + } + + return nil +} + +// detectMajorVersion queries the database to determine its PostgreSQL major version +func detectMajorVersion(db *sql.DB) (int, error) { + ctx := context.Background() + + // Query PostgreSQL version number (e.g., 170005 for 17.5) + var versionNum int + err := db.QueryRowContext(ctx, "SHOW server_version_num").Scan(&versionNum) + if err != nil { + return 0, fmt.Errorf("failed to query PostgreSQL version: %w", err) + } + + // Extract major version: version_num / 10000 + // e.g., 170005 / 10000 = 17 + majorVersion := versionNum / 10000 + + return majorVersion, nil +} From 65366773f07932b9f97949f0d8192c441dc8dbd2 Mon Sep 17 00:00:00 2001 From: tianzhou Date: Thu, 30 Oct 2025 16:07:11 +0800 Subject: [PATCH 2/3] docs: fix timestamp format documentation in external.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct the comment to match the actual Go time format string. The period (.) is required in Go's time format for fractional seconds, not an underscore. The format "20060102_150405.000000000" produces timestamps like "20251030_160428.458109000". Updated comment from: Format: pgschema_plan_YYYYMMDD_HHMMSS_NNNNNNNNN To: Format: pgschema_plan_YYYYMMDD_HHMMSS.NNNNNNNNN Added clarifying note about Go's time format requirements. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- internal/postgres/external.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/postgres/external.go b/internal/postgres/external.go index 1d12b18d..91b97c09 100644 --- a/internal/postgres/external.go +++ b/internal/postgres/external.go @@ -70,7 +70,8 @@ func NewExternalDatabase(config *ExternalDatabaseConfig) (*ExternalDatabase, err } // Generate temporary schema name with timestamp including nanoseconds for uniqueness - // Format: pgschema_plan_YYYYMMDD_HHMMSS_NNNNNNNNN + // Format: pgschema_plan_YYYYMMDD_HHMMSS.NNNNNNNNN + // Note: The period is required in Go's time format for fractional seconds timestamp := time.Now().Format("20060102_150405.000000000") tempSchema := fmt.Sprintf("pgschema_plan_%s", timestamp) From 3a4070f20b7657d35deee3e59bc7f2f7948f346d Mon Sep 17 00:00:00 2001 From: tianzhou Date: Thu, 30 Oct 2025 16:07:48 +0800 Subject: [PATCH 3/3] chore: update comment --- cmd/plan/plan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/plan/plan.go b/cmd/plan/plan.go index d85d4d77..93abfd0b 100644 --- a/cmd/plan/plan.go +++ b/cmd/plan/plan.go @@ -178,8 +178,8 @@ func CreateDesiredStateProvider(config *PlanConfig) (postgres.DesiredStateProvid return nil, fmt.Errorf("failed to detect PostgreSQL version: %w", err) } - // Extract major version from embedded postgres version string (e.g., "16.9.0" -> 16) - // The version string format is "XX.Y.Z" where XX is the major version + // Extract major version from the target database's version string (e.g., "16.9.0" -> 16). + // The version string format is "XX.Y.Z" where XX is the major version. var targetMajorVersion int _, err = fmt.Sscanf(string(pgVersion), "%d.", &targetMajorVersion) if err != nil {