diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 14e573404..c06e99425 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -2195,6 +2195,9 @@ LANGUAGE plpgsql NO SQL;`) numSegments := dbconn.MustSelectString(restoreConn, "SELECT numsegments FROM gp_distribution_policy where localoid = 'schemaone.test_table'::regclass::oid") Expect(numSegments).To(Equal(strconv.Itoa(segmentCount))) + // check there is no pipe errors on segments + errSegments := dbconn.MustSelectString(restoreConn, fmt.Sprintf("SELECT exists (SELECT * FROM gp_toolkit.__gp_log_segment_ext WHERE logdatabase = current_database() AND logmessage LIKE 'read err msg from pipe%%_%06d_%%')", gprestoreCmd.Process.Pid)) + Expect(errSegments).To(Equal("false")) }, Entry("Can backup a 1-segment cluster and restore to current cluster with replicated tables", "20221104023842", "1-segment-db-replicated"), Entry("Can backup a 3-segment cluster and restore to current cluster with replicated tables", "20221104023611", "3-segment-db-replicated"), diff --git a/restore/data.go b/restore/data.go index 03e2617cb..57e293cd9 100644 --- a/restore/data.go +++ b/restore/data.go @@ -26,7 +26,7 @@ var ( tableDelim = "," ) -func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, tableName string, tableAttributes string, destinationToRead string, singleDataFile bool, whichConn int) (int64, error) { +func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, tableName string, tableAttributes string, destinationToRead string, singleDataFile bool, whichConn int, isReplicated bool) (int64, error) { whichConn = connectionPool.ValidateConnNum(whichConn) copyCommand := "" readFromDestinationCommand := "cat" @@ -50,7 +50,7 @@ func CopyTableIn(queryContext context.Context, connectionPool *dbconn.DBConn, ta // During a larger-to-smaller restore, we need multiple COPY passes to load all the data. // One pass is sufficient for smaller-to-larger and normal restores. batches := 1 - if resizeCluster && origSize > destSize { + if !isReplicated && resizeCluster && origSize > destSize { batches = origSize / destSize if origSize%destSize != 0 { batches += 1 @@ -89,7 +89,7 @@ func restoreSingleTableData(queryContext context.Context, fpInfo *filepath.FileP destinationToRead = fpInfo.GetTableBackupFilePathForCopyCommand(entry.Oid, utils.GetPipeThroughProgram().Extension, backupConfig.SingleDataFile) } gplog.Debug("Reading from %s", destinationToRead) - numRowsRestored, err := CopyTableIn(queryContext, connectionPool, tableName, entry.AttributeString, destinationToRead, backupConfig.SingleDataFile, whichConn) + numRowsRestored, err := CopyTableIn(queryContext, connectionPool, tableName, entry.AttributeString, destinationToRead, backupConfig.SingleDataFile, whichConn, entry.IsReplicated) if err != nil { return err } diff --git a/restore/data_test.go b/restore/data_test.go index c39a9e5ae..6fb13dd80 100644 --- a/restore/data_test.go +++ b/restore/data_test.go @@ -34,7 +34,7 @@ var _ = Describe("restore/data tests", func() { execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | gzip -d -c' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -43,7 +43,7 @@ var _ = Describe("restore/data tests", func() { execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | zstd --decompress -c' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -51,7 +51,7 @@ var _ = Describe("restore/data tests", func() { execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -59,7 +59,7 @@ var _ = Describe("restore/data tests", func() { execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -72,7 +72,7 @@ var _ = Describe("restore/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -85,7 +85,7 @@ var _ = Describe("restore/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -97,7 +97,7 @@ var _ = Describe("restore/data tests", func() { mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 0)) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).ShouldNot(HaveOccurred()) }) @@ -111,7 +111,7 @@ var _ = Describe("restore/data tests", func() { } mock.ExpectExec(execStr).WillReturnError(pgErr) filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" - _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0) + _, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("Error loading data into table public.foo: " + @@ -119,6 +119,227 @@ var _ = Describe("restore/data tests", func() { "ERROR: value of distribution key doesn't belong to segment with ID 0, it belongs to segment with ID 1 (SQLSTATE 22P04)")) }) }) + Describe("CopyTableIn with resize restore from 4 to 3 segments", func() { + BeforeEach(func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "cat", OutputCommand: "cat -", InputCommand: "cat -", Extension: ""}) + backup.SetPluginConfig(nil) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "") + _ = cmdFlags.Set(options.RESIZE_CLUSTER, "true") + backup.SetCluster(&cluster.Cluster{ContentIDs: []int{-1, 0, 1, 2}}) + restore.SetBackupConfig(&history.BackupConfig{SegmentCount: 4}) + }) + It("will restore a table from its own file with gzip compression", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from its own file with zstd compression", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from its own file without compression", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from a single data file", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from its own file with gzip compression using a plugin", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from its own file with zstd compression using a plugin", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will restore a table from its own file without compression using a plugin", func() { + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(20))) + }) + It("will output expected error string from COPY ON SEGMENT failure", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + pgErr := &pgconn.PgError{ + Severity: "ERROR", + Code: "22P04", + Message: "value of distribution key doesn't belong to segment with ID 0, it belongs to segment with ID 1", + Where: "COPY foo, line 1: \"5\"", + } + mock.ExpectExec(execStr).WillReturnError(pgErr) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, false) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Error loading data into table public.foo: " + + "COPY foo, line 1: \"5\": " + + "ERROR: value of distribution key doesn't belong to segment with ID 0, it belongs to segment with ID 1 (SQLSTATE 22P04)")) + Expect(numRowsRestored).Should(Equal(int64(0))) + }) + }) + Describe("CopyTableIn with resize restore from 4 to 3 segments and replicated table", func() { + BeforeEach(func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "cat", OutputCommand: "cat -", InputCommand: "cat -", Extension: ""}) + backup.SetPluginConfig(nil) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "") + _ = cmdFlags.Set(options.RESIZE_CLUSTER, "true") + backup.SetCluster(&cluster.Cluster{ContentIDs: []int{-1, 0, 1, 2}}) + restore.SetBackupConfig(&history.BackupConfig{SegmentCount: 4}) + }) + It("will restore a table from its own file with gzip compression", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from its own file with zstd compression", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456.zst" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from its own file without compression", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from a single data file", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, true, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from its own file with gzip compression using a plugin", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "gzip", OutputCommand: "gzip -c -1", InputCommand: "gzip -d -c", Extension: ".gz"}) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from its own file with zstd compression using a plugin", func() { + utils.SetPipeThroughProgram(utils.PipeThroughProgram{Name: "zstd", OutputCommand: "zstd --compress -1 -c", InputCommand: "zstd --decompress -c", Extension: ".zst"}) + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.zst" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will restore a table from its own file without compression using a plugin", func() { + _ = cmdFlags.Set(options.PLUGIN_CONFIG, "/tmp/plugin_config") + pluginConfig := utils.PluginConfig{ExecutablePath: "/tmp/fake-plugin.sh", ConfigPath: "/tmp/plugin_config"} + restore.SetPluginConfig(&pluginConfig) + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz | cat -' WITH CSV DELIMITER ',' ON SEGMENT;") + mock.ExpectExec(execStr).WillReturnResult(sqlmock.NewResult(10, 10)) + + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_pipe_3456.gz" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).ShouldNot(HaveOccurred()) + Expect(numRowsRestored).Should(Equal(int64(10))) + }) + It("will output expected error string from COPY ON SEGMENT failure", func() { + execStr := regexp.QuoteMeta("COPY public.foo(i,j) FROM PROGRAM 'cat /backups/20170101/20170101010101/gpbackup__20170101010101_3456 | cat -' WITH CSV DELIMITER ',' ON SEGMENT") + pgErr := &pgconn.PgError{ + Severity: "ERROR", + Code: "22P04", + Message: "value of distribution key doesn't belong to segment with ID 0, it belongs to segment with ID 1", + Where: "COPY foo, line 1: \"5\"", + } + mock.ExpectExec(execStr).WillReturnError(pgErr) + filename := "/backups/20170101/20170101010101/gpbackup__20170101010101_3456" + numRowsRestored, err := restore.CopyTableIn(context.Background(), connectionPool, "public.foo", "(i,j)", filename, false, 0, true) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Error loading data into table public.foo: " + + "COPY foo, line 1: \"5\": " + + "ERROR: value of distribution key doesn't belong to segment with ID 0, it belongs to segment with ID 1 (SQLSTATE 22P04)")) + Expect(numRowsRestored).Should(Equal(int64(0))) + }) + }) Describe("CheckRowsRestored", func() { var ( expectedRows int64 = 10