Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ADBDEV-4377: Fix backup statistics for leaf partitions #42

Merged
merged 2 commits into from
Nov 27, 2023
Merged

ADBDEV-4377: Fix backup statistics for leaf partitions #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bd48761
Fix backup statistics for leaf partitions
RekGRpth Oct 28, 2023
a38e497
Merge branch 'master' into ADBDEV-4377
RekGRpth Nov 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backup/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func DoBackup() {
}

gplog.Info("Gathering table state information")
metadataTables, dataTables := RetrieveAndProcessTables()
metadataTables, dataTables, allTables := RetrieveAndProcessTables()
dataTables, numExtOrForeignTables := GetBackupDataSet(dataTables)
if len(dataTables) == 0 && !backupReport.MetadataOnly {
gplog.Warn("No tables in backup set contain data. Performing metadata-only backup instead.")
Expand Down Expand Up @@ -191,7 +191,7 @@ func DoBackup() {

printDataBackupWarnings(numExtOrForeignTables)
if MustGetFlagBool(options.WITH_STATS) {
backupStatistics(metadataTables)
backupStatistics(allTables)
}

globalTOC.WriteToFileAndMakeReadOnly(globalFPInfo.GetTOCFilePath())
Expand Down
8 changes: 4 additions & 4 deletions backup/wrappers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ func createBackupDirectoriesOnAllHosts() {
* Metadata retrieval wrapper functions
*/

func RetrieveAndProcessTables() ([]Table, []Table) {
func RetrieveAndProcessTables() ([]Table, []Table, []Table) {
quotedIncludeRelations, err := options.QuoteTableNames(connectionPool, MustGetFlagStringArray(options.INCLUDE_RELATION))
gplog.FatalOnError(err)

Expand All @@ -215,12 +215,12 @@ func RetrieveAndProcessTables() ([]Table, []Table) {
tableRelations = append(tableRelations, GetForeignTableRelations(connectionPool)...)
}

tables := ConstructDefinitionsForTables(connectionPool, tableRelations)
allTables := ConstructDefinitionsForTables(connectionPool, tableRelations)

metadataTables, dataTables := SplitTablesByPartitionType(tables, quotedIncludeRelations)
metadataTables, dataTables := SplitTablesByPartitionType(allTables, quotedIncludeRelations)
objectCounts["Tables"] = len(metadataTables)

return metadataTables, dataTables
return metadataTables, dataTables, allTables
}

func retrieveFunctions(sortables *[]Sortable, metadataMap MetadataMap) ([]Function, map[uint32]FunctionInfo) {
Expand Down
16 changes: 3 additions & 13 deletions end_to_end/end_to_end_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1515,13 +1515,6 @@ var _ = Describe("backup and restore end to end tests", func() {
skipIfOldBackupVersionBefore("1.18.0")

testhelper.AssertQueryRuns(backupConn, `
CREATE TABLE et (
id character varying(13),
flg smallint,
dttm timestamp without time zone,
src character varying(80)
) WITH (appendonly='true', orientation='row', compresstype=zstd, compresslevel='3') DISTRIBUTED BY (id);

red1452 marked this conversation as resolved.
Show resolved Hide resolved
CREATE TABLE pt (
id character varying(13),
flg smallint,
Expand All @@ -1532,19 +1525,16 @@ var _ = Describe("backup and restore end to end tests", func() {
);

INSERT INTO pt(id, flg, dttm, src) VALUES (1, 1, now(), 'val');
INSERT INTO et(id, flg, dttm, src) VALUES (2, 2, now(), 'val');

ANALYZE pt;
ANALYZE et;
ANALYZE ROOTPARTITION pt;

ALTER TABLE pt EXCHANGE PARTITION src_mdm WITH TABLE et;
`)

defer testhelper.AssertQueryRuns(backupConn,
`DROP TABLE et CASCADE; DROP TABLE pt CASCADE;`)
`DROP TABLE pt CASCADE;`)
timestamp := gpbackup(gpbackupPath, backupHelperPath,
"--with-stats",
"--leaf-partition-data",
"--backup-dir", backupDir)
files, err := path.Glob(path.Join(backupDir, "*-1/backups/*",
timestamp, "*statistics.sql"))
Expand All @@ -1562,7 +1552,7 @@ var _ = Describe("backup and restore end to end tests", func() {
assertPGClassStatsRestored(backupConn, restoreConn, publicSchemaTupleCounts)
assertPGClassStatsRestored(backupConn, restoreConn, schema2TupleCounts)

statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where cl.relname != 'pt_1_prt_src_mdm' AND %s;`, backup.SchemaFilterClause("nm"))
statsQuery := fmt.Sprintf(`SELECT count(*) AS string FROM pg_statistic st left join pg_class cl on st.starelid = cl.oid left join pg_namespace nm on cl.relnamespace = nm.oid where %s;`, backup.SchemaFilterClause("nm"))
backupStatisticCount := dbconn.MustSelectString(backupConn, statsQuery)
restoredStatisticsCount := dbconn.MustSelectString(restoreConn, statsQuery)

Expand Down
20 changes: 10 additions & 10 deletions integration/wrappers_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ var _ = Describe("Wrappers Integration", func() {
connectionPool.MustBegin(0)
defer connectionPool.MustCommit(0)

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("foo"))
Expect(dataTables[1].Name).To(Equal(`"BAR"`))
Expand All @@ -59,7 +59,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -72,7 +72,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -84,7 +84,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("ten"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -115,7 +115,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand All @@ -127,7 +127,7 @@ var _ = Describe("Wrappers Integration", func() {
backup.DoInit(rootCmd) // initialize the ObjectCount
rootCmd.Execute()

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("thousands"))
Expect(dataTables[1].Name).To(Equal("empty"))
Expand Down Expand Up @@ -164,7 +164,7 @@ var _ = Describe("Wrappers Integration", func() {
err := opts.ExpandIncludesForPartitions(connectionPool, rootCmd.Flags())
Expect(err).ShouldNot(HaveOccurred())

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(2))
Expect(dataTables[0].Name).To(Equal("partition_table"))
Expect(dataTables[1].Name).To(Equal("ten"))
Expand All @@ -180,7 +180,7 @@ var _ = Describe("Wrappers Integration", func() {
err := opts.ExpandIncludesForPartitions(connectionPool, rootCmd.Flags())
Expect(err).ShouldNot(HaveOccurred())

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()

Expect(len(dataTables)).To(Equal(3))
Expect(dataTables[0].Name).To(Equal("partition_table_1_prt_girls"))
Expand Down Expand Up @@ -211,7 +211,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand All @@ -230,7 +230,7 @@ var _ = Describe("Wrappers Integration", func() {
rootCmd := &cobra.Command{}
backup.DoInit(rootCmd) // initialize the ObjectCount

_, dataTables := backup.RetrieveAndProcessTables()
_, dataTables, _ := backup.RetrieveAndProcessTables()
Expect(len(dataTables)).To(Equal(3))

Expect(dataTables[0].Name).To(Equal("thousands"))
Expand Down
Loading