Initial implementation for Apple config profiles.

changes/23238-use-secrets-in-scripts-profiles

@@ -2,3 +2,4 @@ Added ability to use secrets ($FLEET_SECRET_YOURNAME) in scripts and profiles.
 - Added `/fleet/spec/secret_variables` API endpoint.
 - fleetctl gitops identifies secrets in scripts and profiles and saves them on the Fleet server.
 - secret values are populated when scripts and profiles are sent to devices.
+- When fleetctl gitops updates profiles, if the secret value has changed, the profile is updated on the host.

server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go

@@ -0,0 +1,51 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20241223115925, Down_20241223115925)
+}
+
+func Up_20241223115925(tx *sql.Tx) error {
+	// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
+	_, err := tx.Exec(`ALTER TABLE mdm_apple_configuration_profiles
+    	ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+		MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+		MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to alter mdm_apple_configuration_profiles table: %w", err)
+	}
+
+	// Add secrets_updated_at column to host_mdm_apple_profiles
+	_, err = tx.Exec(`ALTER TABLE host_mdm_apple_profiles
+    	ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`)
+	if err != nil {
+		return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_profiles table: %w", err)
+	}
+
+	// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
+	_, err = tx.Exec(`ALTER TABLE mdm_apple_declarations
+    	ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+		MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+		MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to alter mdm_apple_declarations table: %w", err)
+	}
+
+	// Add secrets_updated_at column to host_mdm_apple_declarations
+	_, err = tx.Exec(`ALTER TABLE host_mdm_apple_declarations
+		-- defaulting to NULL for backward compatibility with existing declarations
+    	ADD COLUMN secrets_updated_at TIMESTAMP(6) NULL`)
+	if err != nil {
+		return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_declarations table: %w", err)
+	}
+
+	return nil
+}
+
+func Down_20241223115925(_ *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/secret_variables.go

@@ -2,8 +2,11 @@ package mysql
 
 import (
 	"context"
+	"database/sql"
+	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/fleetdm/fleet/v4/server/contexts/ctxerr"
 	"github.com/fleetdm/fleet/v4/server/fleet"
@@ -15,24 +18,69 @@ func (ds *Datastore) UpsertSecretVariables(ctx context.Context, secretVariables
 		return nil
 	}
 
-	values := strings.TrimSuffix(strings.Repeat("(?,?),", len(secretVariables)), ",")
+	// The secret variables should rarely change, so we do not use a transaction here.
+	// When we encrypt a secret variable, it is salted, so the encrypted data is different each time.
+	// In order to keep the updated_at timestamp correct, we need to compare the encrypted value
+	// with the existing value in the database. If the values are the same, we do not update the row.
 
-	stmt := fmt.Sprintf(`
-		INSERT INTO secret_variables (name, value)
-		VALUES %s
-		ON DUPLICATE KEY UPDATE value = VALUES(value)`, values)
-
-	args := make([]interface{}, 0, len(secretVariables)*2)
+	var names []string
 	for _, secretVariable := range secretVariables {
-		valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
-		if err != nil {
-			return ctxerr.Wrap(ctx, err, "encrypt secret value with server private key")
+		names = append(names, secretVariable.Name)
+	}
+	existingVariables, err := ds.GetSecretVariables(ctx, names)
+	if err != nil {
+		return ctxerr.Wrap(ctx, err, "get existing secret variables")
+	}
+	existingVariableMap := make(map[string]string, len(existingVariables))
+	for _, existingVariable := range existingVariables {
+		existingVariableMap[existingVariable.Name] = existingVariable.Value
+	}
+	var variablesToInsert []fleet.SecretVariable
+	var variablesToUpdate []fleet.SecretVariable
+	for _, secretVariable := range secretVariables {
+		existingValue, ok := existingVariableMap[secretVariable.Name]
+		switch {
+		case !ok:
+			variablesToInsert = append(variablesToInsert, secretVariable)
+		case existingValue != secretVariable.Value:
+			variablesToUpdate = append(variablesToUpdate, secretVariable)
+		default:
+			// No change -- the variable value is the same
 		}
-		args = append(args, secretVariable.Name, valueEncrypted)
 	}
 
-	if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
-		return ctxerr.Wrap(ctx, err, "upsert secret variables")
+	if len(variablesToInsert) > 0 {
+		values := strings.TrimSuffix(strings.Repeat("(?,?),", len(variablesToInsert)), ",")
+		stmt := fmt.Sprintf(`
+		INSERT INTO secret_variables (name, value)
+		VALUES %s`, values)
+		args := make([]interface{}, 0, len(variablesToInsert)*2)
+		for _, secretVariable := range variablesToInsert {
+			valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
+			if err != nil {
+				return ctxerr.Wrap(ctx, err, "encrypt secret value for insert with server private key")
+			}
+			args = append(args, secretVariable.Name, valueEncrypted)
+		}
+		if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil {
+			return ctxerr.Wrap(ctx, err, "insert secret variables")
+		}
+	}
+
+	if len(variablesToUpdate) > 0 {
+		stmt := `
+		UPDATE secret_variables
+		SET value = ?
+		WHERE name = ?`
+		for _, secretVariable := range variablesToUpdate {
+			valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey)
+			if err != nil {
+				return ctxerr.Wrap(ctx, err, "encrypt secret value for update with server private key")
+			}
+			if _, err := ds.writer(ctx).ExecContext(ctx, stmt, valueEncrypted, secretVariable.Name); err != nil {
+				return ctxerr.Wrap(ctx, err, "update secret variables")
+			}
+		}
 	}
 
 	return nil
@@ -69,6 +117,31 @@ func (ds *Datastore) GetSecretVariables(ctx context.Context, names []string) ([]
 	return secretVariables, nil
 }
 
+func (ds *Datastore) secretVariablesUpdated(ctx context.Context, q sqlx.QueryerContext, names []string, timestamp time.Time) (bool, error) {
+	if len(names) == 0 {
+		return false, nil
+	}
+
+	stmt, args, err := sqlx.In(`
+		SELECT 1
+		FROM secret_variables
+		WHERE name IN (?) AND updated_at > ?`, names, timestamp)
+	if err != nil {
+		return false, ctxerr.Wrap(ctx, err, "build secret variables query")
+	}
+
+	var updated bool
+	err = sqlx.GetContext(ctx, q, &updated, stmt, args...)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		return false, nil
+	case err != nil:
+		return false, ctxerr.Wrap(ctx, err, "get secret variables")
+	default:
+		return updated, nil
+	}
+}
+
 func (ds *Datastore) ExpandEmbeddedSecrets(ctx context.Context, document string) (string, error) {
 	embeddedSecrets := fleet.ContainsPrefixVars(document, fleet.ServerSecretPrefix)
 	if len(embeddedSecrets) == 0 {

server/datastore/mysql/secret_variables_test.go

@@ -3,8 +3,10 @@ package mysql
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/jmoiron/sqlx"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -59,18 +61,35 @@ func testUpsertSecretVariables(t *testing.T, ds *Datastore) {
 		assert.Equal(t, secretMap[result.Name], result.Value)
 	}
 
-	// Update a secret
+	// Update a secret and insert a new one
 	secretMap["test2"] = "newTestValue2"
+	secretMap["test4"] = "testValue4"
 	err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{
 		{Name: "test2", Value: secretMap["test2"]},
+		{Name: "test4", Value: secretMap["test4"]},
 	})
 	assert.NoError(t, err)
-	results, err = ds.GetSecretVariables(ctx, []string{"test2"})
+	results, err = ds.GetSecretVariables(ctx, []string{"test2", "test4"})
 	assert.NoError(t, err)
-	require.Len(t, results, 1)
-	assert.Equal(t, "test2", results[0].Name)
-	assert.Equal(t, secretMap[results[0].Name], results[0].Value)
+	require.Len(t, results, 2)
+	for _, result := range results {
+		assert.Equal(t, secretMap[result.Name], result.Value)
+	}
 
+	// Make sure updated_at timestamp does not change when we update a secret with the same value
+	var myTime time.Time
+	ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+		return sqlx.GetContext(ctx, q, &myTime, "SELECT updated_at FROM secret_variables WHERE name = ?", "test1")
+	})
+	err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{
+		{Name: "test1", Value: secretMap["test1"]},
+	})
+	assert.NoError(t, err)
+	var newTime time.Time
+	ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error {
+		return sqlx.GetContext(ctx, q, &newTime, "SELECT updated_at FROM secret_variables WHERE name = ?", "test1")
+	})
+	assert.Equal(t, myTime, newTime)
 }
 
 func testValidateEmbeddedSecrets(t *testing.T, ds *Datastore) {

server/fleet/apple_mdm.go

@@ -1,7 +1,6 @@
 package fleet
 
 import (
-	"bytes"
 	"context"
 	"crypto/md5" // nolint: gosec
 	"encoding/hex"
@@ -205,6 +204,7 @@ type MDMAppleConfigProfile struct {
 	LabelsExcludeAny []ConfigurationProfileLabel `db:"-" json:"labels_exclude_any,omitempty"`
 	CreatedAt        time.Time                   `db:"created_at" json:"created_at"`
 	UploadedAt       time.Time                   `db:"uploaded_at" json:"updated_at"` // NOTE: JSON field is still `updated_at` for historical reasons, would be an API breaking change
+	SecretsUpdatedAt time.Time                   `db:"secrets_updated_at" json:"-"`
 }
 
 // MDMProfilesUpdates flags updates that were done during batch processing of profiles.
@@ -321,6 +321,7 @@ type MDMAppleProfilePayload struct {
 	HostUUID          string             `db:"host_uuid"`
 	HostPlatform      string             `db:"host_platform"`
 	Checksum          []byte             `db:"checksum"`
+	SecretsUpdatedAt  time.Time          `db:"secrets_updated_at"`
 	Status            *MDMDeliveryStatus `db:"status" json:"status"`
 	OperationType     MDMOperationType   `db:"operation_type"`
 	Detail            string             `db:"detail"`
@@ -333,20 +334,6 @@ func (p *MDMAppleProfilePayload) DidNotInstallOnHost() bool {
 	return p.Status != nil && (*p.Status == MDMDeliveryFailed || *p.Status == MDMDeliveryPending) && p.OperationType == MDMOperationTypeInstall
 }
 
-func (p MDMAppleProfilePayload) Equal(other MDMAppleProfilePayload) bool {
-	statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status
-	return p.ProfileUUID == other.ProfileUUID &&
-		p.ProfileIdentifier == other.ProfileIdentifier &&
-		p.ProfileName == other.ProfileName &&
-		p.HostUUID == other.HostUUID &&
-		p.HostPlatform == other.HostPlatform &&
-		bytes.Equal(p.Checksum, other.Checksum) &&
-		statusEqual &&
-		p.OperationType == other.OperationType &&
-		p.Detail == other.Detail &&
-		p.CommandUUID == other.CommandUUID
-}
-
 type MDMAppleBulkUpsertHostProfilePayload struct {
 	ProfileUUID       string
 	ProfileIdentifier string
@@ -697,18 +684,9 @@ type MDMAppleHostDeclaration struct {
 	// Checksum contains the MD5 checksum of the declaration JSON uploaded
 	// by the IT admin. Fleet uses this value as the ServerToken.
 	Checksum string `db:"checksum" json:"-"`
-}
 
-func (p MDMAppleHostDeclaration) Equal(other MDMAppleHostDeclaration) bool {
-	statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status
-	return statusEqual &&
-		p.HostUUID == other.HostUUID &&
-		p.DeclarationUUID == other.DeclarationUUID &&
-		p.Name == other.Name &&
-		p.Identifier == other.Identifier &&
-		p.OperationType == other.OperationType &&
-		p.Detail == other.Detail &&
-		p.Checksum == other.Checksum
+	// SecretsUpdatedAt is the timestamp when the secrets were last updated or when this declaration was uploaded.
+	SecretsUpdatedAt time.Time `db:"secrets_updated_at" json:"-"`
 }
 
 func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration {