PMM-11180 upgrade matrix

.github/workflows/agent.yml

@@ -28,15 +28,18 @@ jobs:
     strategy:
       matrix:
         images:
-          - { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'mysql:5.7', mongo: 'mongo:4.4', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'mysql:8.0', mongo: 'mongo:4.4', postgres: 'postgres:17rc1', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:5.6', mongo: 'mongo:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:5.7', mongo: 'mongo:5.0', postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:8.0', mongo: 'mongo:6.0', postgres: 'postgres:15', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:lts', mongo: 'mongo:7.0', postgres: 'postgres:16', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mysql:latest', mongo: 'mongo:latest', postgres: 'postgres:latest', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
 
           # Percona + latest PMM Server release
-          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:17-dev', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:5.6', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:14', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:5.7', mongo: 'percona/percona-server-mongodb:5.0', postgres: 'perconalab/percona-distribution-postgresql:15', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:8.0', mongo: 'percona/percona-server-mongodb:6.0', postgres: 'perconalab/percona-distribution-postgresql:16', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:8', mongo: 'percona/percona-server-mongodb:7.0', postgres: 'perconalab/percona-distribution-postgresql:17', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'percona:latest', mongo: 'percona/percona-server-mongodb:latest', postgres: 'perconalab/percona-distribution-postgresql:latest', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
 
           # MariaDB (only 3 latest GA versions)
           #   + older supported MongoDB versions
@@ -46,9 +49,9 @@ jobs:
           # - MYSQL_IMAGE=mariadb:10.0
           # - MYSQL_IMAGE=mariadb:10.1
 
-          - { mysql: 'mariadb:10.2', mongo: 'mongo:4.4',                          postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'mariadb:10.3', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
-          - { mysql: 'mariadb:10.4',                                              postgres: 'postgres:14', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mariadb:11.2', mongo: 'mongo:4.4',                          postgres: 'postgres:12', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mariadb:11.4', mongo: 'percona/percona-server-mongodb:4.4', postgres: 'perconalab/percona-distribution-postgresql:13', pg_libs: 'pg_stat_statements,pg_stat_monitor', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
+          - { mysql: 'mariadb:11.5',                                              postgres: 'postgres:13', pmm_server: 'perconalab/pmm-server:3-dev-latest' }
 
     continue-on-error: true
 
@@ -120,3 +123,5 @@ jobs:
           echo "--- GO Environment ---"
           go env | sort
           git status
+          docker compose logs
+          docker compose ps -a

agent/agents/mongodb/internal/profiler/profiler_test.go

@@ -16,9 +16,7 @@ package profiler
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"os"
 	"sort"
 	"strings"
 	"testing"
@@ -32,7 +30,6 @@ import (
 
 	"github.com/percona/pmm/agent/agents/mongodb/internal/profiler/aggregator"
 	"github.com/percona/pmm/agent/agents/mongodb/internal/report"
-	"github.com/percona/pmm/agent/runner/actions"
 	"github.com/percona/pmm/agent/utils/templates"
 	"github.com/percona/pmm/agent/utils/tests"
 	"github.com/percona/pmm/agent/utils/truncate"
@@ -69,11 +66,11 @@ func TestProfiler(t *testing.T) {
 	tempDir := t.TempDir()
 	sslDSN, err := templates.RenderDSN(sslDSNTemplate, files, tempDir)
 	require.NoError(t, err)
-	for _, url := range []string{
-		"mongodb://root:root-password@127.0.0.1:27017/admin",
-		sslDSN,
+	for name, url := range map[string]string{
+		"normal": tests.GetTestMongoDBDSN(t),
+		"ssl":    sslDSN,
 	} {
-		t.Run(url, func(t *testing.T) {
+		t.Run(name, func(t *testing.T) {
 			testProfiler(t, url)
 		})
 	}
@@ -210,47 +207,6 @@ func testProfiler(t *testing.T, url string) {
 	require.NotNil(t, findBucket)
 	assert.Equal(t, "FIND people name_00\ufffd", findBucket.Common.Fingerprint)
 	assert.Equal(t, docsCount, findBucket.Mongodb.MDocsReturnedSum)
-
-	// PMM-4192 This seems to be out of place because it is an Explain test but there was a problem with
-	// the new MongoDB driver and bson.D and we were capturing invalid queries in the profiler.
-	// This test is here to ensure the query example the profiler captures is valid to be used in Explain.
-	t.Run("TestMongoDBExplain", func(t *testing.T) {
-		id := "abcd1234"
-
-		params := &agentv1.StartActionRequest_MongoDBExplainParams{
-			Dsn:   tests.GetTestMongoDBDSN(t),
-			Query: findBucket.Common.Example,
-		}
-
-		ex, err := actions.NewMongoDBExplainAction(id, 5*time.Second, params, os.TempDir())
-		require.NoError(t, err)
-
-		ctx, cancel := context.WithTimeout(context.Background(), ex.Timeout())
-		defer cancel()
-		res, err := ex.Run(ctx)
-		assert.Nil(t, err)
-
-		want := map[string]interface{}{
-			"indexFilterSet": false,
-			"namespace":      "test_00.people",
-			"parsedQuery": map[string]interface{}{
-				"name_00\ufffd": map[string]interface{}{
-					"$eq": "value_00\ufffd",
-				},
-			},
-			"plannerVersion": map[string]interface{}{"$numberInt": "1"},
-			"rejectedPlans":  []interface{}{},
-		}
-
-		explainM := make(map[string]interface{})
-		err = json.Unmarshal(res, &explainM)
-		assert.Nil(t, err)
-		queryPlanner, ok := explainM["queryPlanner"].(map[string]interface{})
-		want["winningPlan"] = queryPlanner["winningPlan"]
-		assert.Equal(t, ok, true)
-		assert.NotEmpty(t, queryPlanner)
-		assert.Equal(t, want, queryPlanner)
-	})
 }
 
 func cleanUpDBs(t *testing.T, sess *mongo.Client) {

agent/agents/mysql/perfschema/history.go

@@ -43,8 +43,12 @@ func newHistoryCache(typ historyMap, retain time.Duration, sizeLimit uint, l *lo
 	return &historyCache{c}, err
 }
 
-func getHistory(q *reform.Querier) (historyMap, error) {
-	rows, err := q.SelectRows(eventsStatementsHistoryView, "WHERE DIGEST IS NOT NULL AND SQL_TEXT IS NOT NULL")
+func getHistory(q *reform.Querier, long *bool) (historyMap, error) {
+	view := eventsStatementsHistoryView
+	if long != nil && *long {
+		view = eventsStatementsHistoryLongView
+	}
+	rows, err := q.SelectRows(view, "WHERE DIGEST IS NOT NULL AND SQL_TEXT IS NOT NULL")
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to query events_statements_history")
 	}

agent/agents/mysql/perfschema/models.go

@@ -14,6 +14,8 @@
 
 package perfschema
 
+import "gopkg.in/reform.v1/parse"
+
 //go:generate ../../../../bin/reform
 
 // eventsStatementsSummaryByDigest represents a row in performance_schema.events_statements_summary_by_digest table.
@@ -121,3 +123,20 @@ type setupInstruments struct {
 	Enabled string  `reform:"ENABLED"`
 	Timed   *string `reform:"TIMED"` // nullable in 8.0
 }
+
+// eventsStatementsHistoryView represents events_statements_history view or table in SQL database.
+var eventsStatementsHistoryLongView = &eventsStatementsHistoryViewType{
+	s: parse.StructInfo{
+		Type:      "eventsStatementsHistory",
+		SQLSchema: "performance_schema",
+		SQLName:   "events_statements_history_long",
+		Fields: []parse.FieldInfo{
+			{Name: "SQLText", Type: "*string", Column: "SQL_TEXT"},
+			{Name: "Digest", Type: "*string", Column: "DIGEST"},
+			{Name: "DigestText", Type: "*string", Column: "DIGEST_TEXT"},
+			{Name: "CurrentSchema", Type: "*string", Column: "CURRENT_SCHEMA"},
+		},
+		PKFieldIndex: -1,
+	},
+	z: (&eventsStatementsHistory{}).Values(),
+}

agent/agents/mysql/perfschema/perfschema.go

@@ -35,6 +35,7 @@ import (
 	"github.com/percona/pmm/agent/queryparser"
 	"github.com/percona/pmm/agent/tlshelpers"
 	"github.com/percona/pmm/agent/utils/truncate"
+	"github.com/percona/pmm/agent/utils/version"
 	agentv1 "github.com/percona/pmm/api/agent/v1"
 	inventoryv1 "github.com/percona/pmm/api/inventory/v1"
 	"github.com/percona/pmm/utils/sqlmetrics"
@@ -67,6 +68,7 @@ type PerfSchema struct {
 	changes                chan agents.Change
 	historyCache           *historyCache
 	summaryCache           *summaryCache
+	useLong                *bool
 }
 
 // Params represent Agent parameters.
@@ -274,7 +276,14 @@ func (m *PerfSchema) runHistoryCacheRefresher(ctx context.Context) {
 }
 
 func (m *PerfSchema) refreshHistoryCache() error {
-	current, err := getHistory(m.q)
+	if m.useLong == nil {
+		sqlVersion, vendor, err := version.GetMySQLVersion(context.Background(), m.q)
+		if err != nil {
+			return errors.Wrap(err, "cannot get MySQL version")
+		}
+		m.useLong = pointer.ToBool(vendor == version.MariaDBVendor && sqlVersion.Float() >= 11)
+	}
+	current, err := getHistory(m.q, m.useLong)
 	if err != nil {
 		return err
 	}

agent/agents/mysql/perfschema/perfschema_test.go

@@ -201,6 +201,15 @@ func setup(t *testing.T, sp *setupParams) *PerfSchema {
 
 // filter removes buckets for queries that are not expected by tests.
 func filter(mb []*agentv1.MetricsBucket) []*agentv1.MetricsBucket {
+	filterList := map[string]struct{}{
+		"ANALYZE TABLE `city`":                            {}, // OpenTestMySQL
+		"SHOW GLOBAL VARIABLES WHERE `Variable_name` = ?": {}, // MySQLVersion
+		"SHOW VARIABLES LIKE ?":                           {}, // MariaDBVersion
+		"SELECT `id` FROM `city` LIMIT ?":                 {}, // waitForFixtures
+		"SELECT ID FROM `city` LIMIT ?":                   {}, // waitForFixtures for MariaDB
+		"SELECT COUNT ( * ) FROM `city`":                  {}, // actions tests
+		"CREATE TABLE IF NOT EXISTS `t1` ( `col1` CHARACTER (?) ) CHARACTER SET `utf8mb4` COLLATE `utf8mb4_general_ci`": {}, // tests for invalid characters
+	}
 	res := make([]*agentv1.MetricsBucket, 0, len(mb))
 	for _, b := range mb {
 		switch {
@@ -215,23 +224,14 @@ func filter(mb []*agentv1.MetricsBucket) []*agentv1.MetricsBucket {
 			continue
 		case strings.Contains(b.Common.Example, "/* pmm-agent-tests:waitForFixtures */"):
 			continue
-		}
-
-		switch {
-		case b.Common.Fingerprint == "ANALYZE TABLE `city`": // OpenTestMySQL
-			continue
-		case b.Common.Fingerprint == "SHOW GLOBAL VARIABLES WHERE `Variable_name` = ?": // MySQLVersion
-			continue
-		case b.Common.Fingerprint == "SELECT `id` FROM `city` LIMIT ?": // waitForFixtures
+		case strings.Contains(b.Common.Fingerprint, "events_statements_history"):
 			continue
-		case b.Common.Fingerprint == "SELECT ID FROM `city` LIMIT ?": // waitForFixtures for MariaDB
-			continue
-		case b.Common.Fingerprint == "SELECT COUNT ( * ) FROM `city`": // actions tests
+		case strings.HasPrefix(b.Common.Fingerprint, "SELECT @@`slow_query_log"): // slowlog
 			continue
-		case b.Common.Fingerprint == "CREATE TABLE IF NOT EXISTS `t1` ( `col1` CHARACTER (?) ) CHARACTER SET `utf8mb4` COLLATE `utf8mb4_general_ci`": // tests for invalid characters
+		case strings.HasPrefix(b.Common.Fingerprint, "TRUNCATE"): // OpenTestMySQL
 			continue
-
-		case strings.HasPrefix(b.Common.Fingerprint, "SELECT @@`slow_query_log"): // slowlog
+		}
+		if _, ok := filterList[b.Common.Fingerprint]; ok {
 			continue
 		}
 
@@ -246,7 +246,7 @@ func TestPerfSchema(t *testing.T) {
 	db := reform.NewDB(sqlDB, mysql.Dialect, reform.NewPrintfLogger(t.Logf))
 
 	updateQuery := fmt.Sprintf("UPDATE /* %s */ ", queryTag)
-	_, err := db.Exec(updateQuery + "performance_schema.setup_consumers SET ENABLED='YES' WHERE NAME='events_statements_history'")
+	_, err := db.Exec(updateQuery + "performance_schema.setup_consumers SET ENABLED='YES'")
 	require.NoError(t, err, "failed to enable events_statements_history consumer")
 
 	structs, err := db.SelectAllFrom(setupConsumersView, "ORDER BY NAME")
@@ -259,6 +259,7 @@ func TestPerfSchema(t *testing.T) {
 	var rowsExamined float32
 	ctx := context.Background()
 	mySQLVersion, mySQLVendor, _ := version.GetMySQLVersion(ctx, db.WithTag("pmm-agent-tests:MySQLVersion"))
+	t.Logf("MySQL version: %s, vendor: %s", mySQLVersion, mySQLVendor)
 	var digests map[string]string // digest_text/fingerprint to digest/query_id
 	switch fmt.Sprintf("%s-%s", mySQLVersion, mySQLVendor) {
 	case "5.6-oracle":
@@ -283,7 +284,7 @@ func TestPerfSchema(t *testing.T) {
 			"SELECT * FROM `city`": "9c799bdb2460f79b3423b77cd10403da",
 		}
 
-	case "8.0-oracle", "8.0-percona":
+	case "8.0-oracle", "8.0-percona", "8.4-oracle", "9.0-oracle", "9.1-oracle":
 		digests = map[string]string{
 			"SELECT `sleep` (?)":   "0b1b1c39d4ee2dda7df2a532d0a23406d86bd34e2cd7f22e3f7e9dedadff9b69",
 			"SELECT * FROM `city`": "950bdc225cf73c9096ba499351ed4376f4526abad3d8ceabc168b6b28cfc9eab",
@@ -308,6 +309,24 @@ func TestPerfSchema(t *testing.T) {
 			"SELECT * FROM `city`": "a65e76b1643273fa3206b11c4f4d8739",
 		}
 
+	case "11.2-mariadb":
+		digests = map[string]string{
+			"SELECT `sleep` (?)":   "ffbde6c4dfda8dff9a4fefd7e8ed648f",
+			"SELECT * FROM `city`": "d0f2ac0577a44d383c5c0480a420caeb",
+		}
+
+	case "11.4-mariadb":
+		digests = map[string]string{
+			"SELECT `sleep` (?)":   "860792b8f3d058489b287e30ccf3beae",
+			"SELECT * FROM `city`": "457a868ea48e4571327914f2831d62f5",
+		}
+
+	case "11.5-mariadb":
+		digests = map[string]string{
+			"SELECT `sleep` (?)":   "860792b8f3d058489b287e30ccf3beae",
+			"SELECT * FROM `city`": "457a868ea48e4571327914f2831d62f5",
+		}
+
 	default:
 		t.Log("Unhandled version, assuming dummy digests.")
 		digests = map[string]string{
@@ -433,9 +452,9 @@ func TestPerfSchema(t *testing.T) {
 
 		require.NoError(t, m.refreshHistoryCache())
 		var example string
-		switch mySQLVersion.String() {
+		switch {
 		// Perf schema truncates queries with non-utf8 characters.
-		case "8.0":
+		case (mySQLVendor == version.PerconaVendor || mySQLVendor == version.OracleVendor) && mySQLVersion.Float() >= 8.0:
 			example = "SELECT /* t1 controller='test' */ * FROM t1 where col1='Bu"
 		default:
 			example = "SELECT /* t1 controller='test' */ * FROM t1 where col1=..."

agent/agents/postgres/pgstatstatements/pgstatstatements_test.go

@@ -120,18 +120,6 @@ func TestPGStatStatementsQAN(t *testing.T) {
 	engineVersion := tests.PostgreSQLVersion(t, sqlDB)
 	var digests map[string]string // digest_text/fingerprint to digest/query_id
 	switch engineVersion {
-	case "9.4":
-		truncatedMSharedBlksHitSum = float32(1007)
-		digests = map[string]string{
-			selectAllCities:     "3239586867",
-			selectAllCitiesLong: "2745128652",
-		}
-	case "9.5", "9.6":
-		truncatedMSharedBlksHitSum = float32(1007)
-		digests = map[string]string{
-			selectAllCities:     "3994135135",
-			selectAllCitiesLong: "2677760328",
-		}
 	case "10":
 		truncatedMSharedBlksHitSum = float32(1007)
 		digests = map[string]string{
@@ -360,7 +348,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 				MSharedBlkReadTimeSum: actual.Postgresql.MSharedBlkReadTimeSum,
 				MLocalBlkReadTimeCnt:  actual.Postgresql.MLocalBlkReadTimeCnt,
 				MLocalBlkReadTimeSum:  actual.Postgresql.MLocalBlkReadTimeSum,
-				MSharedBlksHitCnt:     1,
+				MSharedBlksHitCnt:     actual.Postgresql.MSharedBlksHitCnt,
 				MSharedBlksHitSum:     actual.Postgresql.MSharedBlksHitSum,
 				MRowsCnt:              1,
 				MRowsSum:              499,
@@ -408,16 +396,8 @@ func TestPGStatStatementsQAN(t *testing.T) {
 		t.Logf("Actual:\n%s", tests.FormatBuckets(buckets))
 		require.Len(t, buckets, 1)
 
-		var fingerprint string
-		tables := []string{tableName}
+		fingerprint := fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
 
-		switch engineVersion {
-		case "9.4", "9.5", "9.6":
-			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES (?, ?, ?, ?)`, tableName)
-			tables = []string{}
-		default:
-			fingerprint = fmt.Sprintf(`INSERT /* CheckMBlkReadTime controller='test' */ INTO %s (customer_id, first_name, last_name, active) VALUES ($1, $2, $3, $4)`, tableName)
-		}
 		actual := buckets[0]
 		assert.NotZero(t, actual.Postgresql.MSharedBlkReadTimeSum+actual.Postgresql.MSharedBlkWriteTimeSum)
 		assert.Equal(t, float32(n), actual.Postgresql.MSharedBlkReadTimeCnt+actual.Postgresql.MSharedBlkWriteTimeCnt)
@@ -426,7 +406,7 @@ func TestPGStatStatementsQAN(t *testing.T) {
 				Queryid:             actual.Common.Queryid,
 				Fingerprint:         fingerprint,
 				Database:            "pmm-agent",
-				Tables:              tables,
+				Tables:              []string{tableName},
 				Comments:            map[string]string{"controller": "test"},
 				Username:            "pmm-agent",
 				AgentId:             "agent_id",

agent/connectionchecker/connection_checker_test.go

@@ -232,7 +232,7 @@ func TestConnectionChecker(t *testing.T) {
 				assert.Empty(t, resp.Error)
 			} else {
 				require.NotEmpty(t, resp.Error)
-				assert.Regexp(t, `^`+tt.expectedErr+`$`, resp.Error)
+				assert.Regexp(t, `(?i)^`+tt.expectedErr+`$`, resp.Error)
 			}
 		})
 	}