Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ADBDEV-5541: Backup statistics by one row #77

Merged
merged 20 commits into from
May 27, 2024
Merged

ADBDEV-5541: Backup statistics by one row #77

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
8d14860
backup statistic table by table
RekGRpth Apr 27, 2024
fa47d99
test
RekGRpth Apr 27, 2024
acefcc3
fix
RekGRpth Apr 27, 2024
c12725b
backup statistics by 100 tables at once
RekGRpth Apr 27, 2024
9b1ef6d
config for batch count
RekGRpth Apr 27, 2024
47c52e1
rewrite
RekGRpth May 2, 2024
2627851
Merge branch 'master' into ADBDEV-5541
RekGRpth May 6, 2024
c4a0ade
Merge branch 'master' into ADBDEV-5541
RekGRpth May 6, 2024
66780be
Merge branch 'master' into ADBDEV-5541
RekGRpth May 15, 2024
a2dbbf4
backup statistics by 10000 tables at once
RekGRpth May 15, 2024
453544d
Merge branch 'master' into ADBDEV-5541
RekGRpth May 20, 2024
557c13e
rewrite
RekGRpth May 20, 2024
d89c019
optimize
RekGRpth May 21, 2024
a5f9260
Merge branch 'master' into ADBDEV-5541
RekGRpth May 24, 2024
14c9a7e
remove functions from public access
RekGRpth May 24, 2024
9812a2c
GetTupleStatistics callback
RekGRpth May 24, 2024
81953c1
GetAttributeStatistics callback
RekGRpth May 24, 2024
82ab713
fix missing check
RekGRpth May 24, 2024
95d2c85
typo
RekGRpth May 24, 2024
8682ffc
move PrintStatisticsStatements function to tests
RekGRpth May 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions backup/queries_statistics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type AttributeStatistic struct {
Values5 pq.StringArray `db:"stavalues5"`
}

func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint32][]AttributeStatistic {
func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table, processRow func(attStat *AttributeStatistic)) {
inheritClause := ""
statSlotClause := ""
if connectionPool.Version.AtLeast("6") {
Expand Down Expand Up @@ -119,14 +119,15 @@ func GetAttributeStatistics(connectionPool *dbconn.DBConn, tables []Table) map[u
inheritClause, statSlotClause, statCollationClause,
SchemaFilterClause("n"), utils.SliceToQuotedString(tablenames))

results := make([]AttributeStatistic, 0)
err := connectionPool.Select(&results, query)
rows, err := connectionPool.Query(query)
whitehawk marked this conversation as resolved.
Show resolved Hide resolved
gplog.FatalOnError(err)
stats := make(map[uint32][]AttributeStatistic)
for _, stat := range results {
stats[stat.Oid] = append(stats[stat.Oid], stat)
for rows.Next() {
var attStat AttributeStatistic
err = rows.StructScan(&attStat)
gplog.FatalOnError(err)
processRow(&attStat)
}
return stats
gplog.FatalOnError(rows.Err())
whitehawk marked this conversation as resolved.
Show resolved Hide resolved
}

type TupleStatistic struct {
Expand All @@ -137,7 +138,7 @@ type TupleStatistic struct {
RelTuples float64
}

func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint32]TupleStatistic {
func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table, processRow func(tupleStat *TupleStatistic)) {
tablenames := make([]string, 0)
for _, table := range tables {
tablenames = append(tablenames, table.FQN())
Expand All @@ -155,12 +156,13 @@ func GetTupleStatistics(connectionPool *dbconn.DBConn, tables []Table) map[uint3
ORDER BY n.nspname, c.relname`,
SchemaFilterClause("n"), utils.SliceToQuotedString(tablenames))

results := make([]TupleStatistic, 0)
err := connectionPool.Select(&results, query)
rows, err := connectionPool.Query(query)
gplog.FatalOnError(err)
stats := make(map[uint32]TupleStatistic)
for _, stat := range results {
stats[stat.Oid] = stat
for rows.Next() {
var tupleStat TupleStatistic
err = rows.StructScan(&tupleStat)
gplog.FatalOnError(err)
processRow(&tupleStat)
}
return stats
gplog.FatalOnError(rows.Err())
}
20 changes: 14 additions & 6 deletions backup/statistics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,23 @@ import (
"github.com/lib/pq"
)

func printTupleStatisticsStatementForTable(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, table Table, tupleStat TupleStatistic) {
tupleQuery := GenerateTupleStatisticsQuery(table, tupleStat)
printStatisticsStatementForTable(statisticsFile, tocfile, table, tupleQuery)
}

func printAttributeStatisticsStatementForTable(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, table Table, attStat AttributeStatistic) {
attributeQueries := GenerateAttributeStatisticsQueries(table, attStat)
for _, attrQuery := range attributeQueries {
printStatisticsStatementForTable(statisticsFile, tocfile, table, attrQuery)
}
}

func PrintStatisticsStatements(statisticsFile *utils.FileWithByteCount, tocfile *toc.TOC, tables []Table, attStats map[uint32][]AttributeStatistic, tupleStats map[uint32]TupleStatistic) {
andr-sokolov marked this conversation as resolved.
Show resolved Hide resolved
for _, table := range tables {
tupleQuery := GenerateTupleStatisticsQuery(table, tupleStats[table.Oid])
printStatisticsStatementForTable(statisticsFile, tocfile, table, tupleQuery)
printTupleStatisticsStatementForTable(statisticsFile, tocfile, table, tupleStats[table.Oid])
bimboterminator1 marked this conversation as resolved.
Show resolved Hide resolved
for _, attStat := range attStats[table.Oid] {
attributeQueries := GenerateAttributeStatisticsQueries(table, attStat)
for _, attrQuery := range attributeQueries {
printStatisticsStatementForTable(statisticsFile, tocfile, table, attrQuery)
}
printAttributeStatisticsStatementForTable(statisticsFile, tocfile, table, attStat)
}
}
}
Expand Down
14 changes: 10 additions & 4 deletions backup/wrappers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,11 +772,17 @@ func backupExtendedStatistic(metadataFile *utils.FileWithByteCount) {
*/

func backupTableStatistics(statisticsFile *utils.FileWithByteCount, tables []Table) {
attStats := GetAttributeStatistics(connectionPool, tables)
tupleStats := GetTupleStatistics(connectionPool, tables)

backupSessionGUC(statisticsFile)
PrintStatisticsStatements(statisticsFile, globalTOC, tables, attStats, tupleStats)
tablesMap := make(map[uint32]Table)
for _, table := range tables {
tablesMap[table.Oid] = table
}
GetTupleStatistics(connectionPool, tables, func(tupleStat *TupleStatistic) {
printTupleStatisticsStatementForTable(statisticsFile, globalTOC, tablesMap[tupleStat.Oid], *tupleStat)
})
GetAttributeStatistics(connectionPool, tables, func(attStat *AttributeStatistic) {
printAttributeStatisticsStatementForTable(statisticsFile, globalTOC, tablesMap[attStat.Oid], *attStat)
})
}

func backupIncrementalMetadata() {
Expand Down
40 changes: 32 additions & 8 deletions integration/statistics_create_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,14 @@ var _ = Describe("backup integration tests", func() {
oldTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo", backup.TYPE_RELATION)
tables[0].Oid = oldTableOid

beforeAttStats := backup.GetAttributeStatistics(connectionPool, tables)
beforeTupleStats := backup.GetTupleStatistics(connectionPool, tables)
beforeAttStats := make(map[uint32][]backup.AttributeStatistic)
backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
beforeAttStats[attStat.Oid] = append(beforeAttStats[attStat.Oid], *attStat)
})
beforeTupleStats := make(map[uint32]backup.TupleStatistic)
backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
beforeTupleStats[tupleStat.Oid] = *tupleStat
})
beforeTupleStat := beforeTupleStats[oldTableOid]

// Drop and recreate the table to clear the statistics
Expand All @@ -44,8 +50,14 @@ var _ = Describe("backup integration tests", func() {

newTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo", backup.TYPE_RELATION)
tables[0].Oid = newTableOid
afterAttStats := backup.GetAttributeStatistics(connectionPool, tables)
afterTupleStats := backup.GetTupleStatistics(connectionPool, tables)
afterAttStats := make(map[uint32][]backup.AttributeStatistic)
backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
afterAttStats[attStat.Oid] = append(afterAttStats[attStat.Oid], *attStat)
})
afterTupleStats := make(map[uint32]backup.TupleStatistic)
backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
afterTupleStats[tupleStat.Oid] = *tupleStat
})
afterTupleStat := afterTupleStats[newTableOid]

oldAtts := beforeAttStats[oldTableOid]
Expand Down Expand Up @@ -74,8 +86,14 @@ var _ = Describe("backup integration tests", func() {
oldTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo''\"''''bar", backup.TYPE_RELATION)
tables[0].Oid = oldTableOid

beforeAttStats := backup.GetAttributeStatistics(connectionPool, tables)
beforeTupleStats := backup.GetTupleStatistics(connectionPool, tables)
beforeAttStats := make(map[uint32][]backup.AttributeStatistic)
backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
beforeAttStats[attStat.Oid] = append(beforeAttStats[attStat.Oid], *attStat)
})
beforeTupleStats := make(map[uint32]backup.TupleStatistic)
backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
beforeTupleStats[tupleStat.Oid] = *tupleStat
})
beforeTupleStat := beforeTupleStats[oldTableOid]

// Drop and recreate the table to clear the statistics
Expand All @@ -88,8 +106,14 @@ var _ = Describe("backup integration tests", func() {

newTableOid := testutils.OidFromObjectName(connectionPool, "public", "foo''\"''''bar", backup.TYPE_RELATION)
tables[0].Oid = newTableOid
afterAttStats := backup.GetAttributeStatistics(connectionPool, tables)
afterTupleStats := backup.GetTupleStatistics(connectionPool, tables)
afterAttStats := make(map[uint32][]backup.AttributeStatistic)
backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
afterAttStats[attStat.Oid] = append(afterAttStats[attStat.Oid], *attStat)
})
afterTupleStats := make(map[uint32]backup.TupleStatistic)
backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
afterTupleStats[tupleStat.Oid] = *tupleStat
})
afterTupleStat := afterTupleStats[newTableOid]

oldAtts := beforeAttStats[oldTableOid]
Expand Down
10 changes: 8 additions & 2 deletions integration/statistics_queries_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,10 @@ var _ = Describe("backup integration tests", func() {
})
Describe("GetAttributeStatistics", func() {
It("returns attribute statistics for a table", func() {
attStats := backup.GetAttributeStatistics(connectionPool, tables)
attStats := make(map[uint32][]backup.AttributeStatistic)
backup.GetAttributeStatistics(connectionPool, tables, func(attStat *backup.AttributeStatistic) {
attStats[attStat.Oid] = append(attStats[attStat.Oid], *attStat)
})
Expect(attStats).To(HaveLen(1))
Expect(attStats[tableOid]).To(HaveLen(3))
tableAttStatsI := attStats[tableOid][0]
Expand Down Expand Up @@ -83,7 +86,10 @@ var _ = Describe("backup integration tests", func() {
})
Describe("GetTupleStatistics", func() {
It("returns tuple statistics for a table", func() {
tupleStats := backup.GetTupleStatistics(connectionPool, tables)
tupleStats := make(map[uint32]backup.TupleStatistic)
backup.GetTupleStatistics(connectionPool, tables, func(tupleStat *backup.TupleStatistic) {
tupleStats[tupleStat.Oid] = *tupleStat
})
Expect(tupleStats).To(HaveLen(1))
tableTupleStats := tupleStats[tableOid]

Expand Down
Loading