From af5042717a3593ee6357d9d54e7d62833a5ef4b1 Mon Sep 17 00:00:00 2001 From: Victor Lyuboslavsky Date: Mon, 23 Dec 2024 15:03:28 -0600 Subject: [PATCH] Initial implementation for Apple config profiles. --- changes/23238-use-secrets-in-scripts-profiles | 1 + server/datastore/mysql/apple_mdm.go | 202 ++++++++---------- .../20241223115925_AddSecretsUpdatedAt.go | 51 +++++ server/datastore/mysql/schema.sql | 18 +- server/datastore/mysql/secret_variables.go | 99 +++++++-- .../datastore/mysql/secret_variables_test.go | 29 ++- server/fleet/apple_mdm.go | 30 +-- server/fleet/apple_mdm_test.go | 141 ------------ 8 files changed, 265 insertions(+), 306 deletions(-) create mode 100644 server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go diff --git a/changes/23238-use-secrets-in-scripts-profiles b/changes/23238-use-secrets-in-scripts-profiles index ae55893da731..4df69e0d13cc 100644 --- a/changes/23238-use-secrets-in-scripts-profiles +++ b/changes/23238-use-secrets-in-scripts-profiles @@ -2,3 +2,4 @@ Added ability to use secrets ($FLEET_SECRET_YOURNAME) in scripts and profiles. - Added `/fleet/spec/secret_variables` API endpoint. - fleetctl gitops identifies secrets in scripts and profiles and saves them on the Fleet server. - secret values are populated when scripts and profiles are sent to devices. +- When fleetctl gitops updates profiles, if the secret value has changed, the profile is updated on the host. diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index 578d5c7f7edb..ab009c0e61e4 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -1701,7 +1701,8 @@ func (ds *Datastore) batchSetMDMAppleProfilesDB( SELECT identifier, profile_uuid, - mobileconfig + mobileconfig, + secrets_updated_at FROM mdm_apple_configuration_profiles WHERE @@ -1724,14 +1725,20 @@ INSERT INTO ) VALUES -- see https://stackoverflow.com/a/51393124/1094941 - ( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP() ) + ( CONCAT('a', CONVERT(uuid() USING utf8mb4)), ?, ?, ?, ?, UNHEX(MD5(mobileconfig)), CURRENT_TIMESTAMP(6) ) ON DUPLICATE KEY UPDATE - uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), + uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP(6)), + secrets_updated_at = IF(checksum = VALUES(checksum), secrets_updated_at, CURRENT_TIMESTAMP(6)), checksum = VALUES(checksum), name = VALUES(name), mobileconfig = VALUES(mobileconfig) ` + const secretsUpdatedInProfile = ` +UPDATE mdm_apple_configuration_profiles +SET secrets_updated_at = CURRENT_TIMESTAMP(6) +WHERE team_id = ? AND identifier = ?` + // use a profile team id of 0 if no-team var profTeamID uint if tmID != nil { @@ -1815,7 +1822,30 @@ ON DUPLICATE KEY UPDATE } return false, ctxerr.Wrapf(ctx, err, "insert new/edited profile with identifier %q", p.Identifier) } - updatedDB = updatedDB || insertOnDuplicateDidInsertOrUpdate(result) + wasUpdated := insertOnDuplicateDidInsertOrUpdate(result) + if !wasUpdated { + // If the profile was not updated, check if it contains any secret variables that have been updated. + secretVars := fleet.ContainsPrefixVars(string(p.Mobileconfig), fleet.ServerSecretPrefix) + if len(secretVars) > 0 { + for _, existingP := range existingProfiles { + // Find the matching existing profile by identifier. We assume the team is the same (profTeamID). + if existingP.Identifier == p.Identifier { + wasUpdated, err = ds.secretVariablesUpdated(ctx, tx, secretVars, existingP.SecretsUpdatedAt) + if err != nil { + return false, ctxerr.Wrap(ctx, err, "check if secret variables were updated") + } + if wasUpdated { + _, err = tx.ExecContext(ctx, secretsUpdatedInProfile, profTeamID, p.Identifier) + if err != nil { + return false, ctxerr.Wrap(ctx, err, "update secrets_updated_at") + } + } + break + } + } + } + } + updatedDB = updatedDB || wasUpdated } // build a list of labels so the associations can be batch-set all at once @@ -1988,13 +2018,14 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( ds.host_platform as host_platform, ds.profile_identifier as profile_identifier, ds.profile_name as profile_name, - ds.checksum as checksum + ds.checksum as checksum, + ds.secrets_updated_at as secrets_updated_at, FROM ( %s ) as ds LEFT JOIN host_mdm_apple_profiles hmap ON hmap.profile_uuid = ds.profile_uuid AND hmap.host_uuid = ds.host_uuid WHERE -- profile has been updated - ( hmap.checksum != ds.checksum ) OR + ( hmap.checksum != ds.checksum ) OR ( hmap.secrets_updated_at < ds.secrets_updated_at ) OR -- profiles in A but not in B ( hmap.profile_uuid IS NULL AND hmap.host_uuid IS NULL ) OR -- profiles in A and B but with operation type "remove" @@ -2060,6 +2091,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( narrowByProfiles = "AND hmap.profile_uuid IN (?)" } + // Note: We do not need secrets_updated_at in the remove statement toRemoveStmt := fmt.Sprintf(` SELECT hmap.profile_uuid as profile_uuid, @@ -2176,46 +2208,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( profilesToInsert := make(map[string]*fleet.MDMAppleProfilePayload) executeUpsertBatch := func(valuePart string, args []any) error { - // Check if the update needs to be done at all. - selectStmt := fmt.Sprintf(` - SELECT - host_uuid, - profile_uuid, - profile_identifier, - status, - COALESCE(operation_type, '') AS operation_type, - COALESCE(detail, '') AS detail, - command_uuid, - profile_name, - checksum, - profile_uuid - FROM host_mdm_apple_profiles WHERE (host_uuid, profile_uuid) IN (%s)`, - strings.TrimSuffix(strings.Repeat("(?,?),", len(profilesToInsert)), ",")) - var selectArgs []any - for _, p := range profilesToInsert { - selectArgs = append(selectArgs, p.HostUUID, p.ProfileUUID) - } - var existingProfiles []fleet.MDMAppleProfilePayload - if err := sqlx.SelectContext(ctx, tx, &existingProfiles, selectStmt, selectArgs...); err != nil { - return ctxerr.Wrap(ctx, err, "bulk set pending profile status select existing") - } - var updateNeeded bool - if len(existingProfiles) == len(profilesToInsert) { - for _, exist := range existingProfiles { - insert, ok := profilesToInsert[fmt.Sprintf("%s\n%s", exist.HostUUID, exist.ProfileUUID)] - if !ok || !exist.Equal(*insert) { - updateNeeded = true - break - } - } - } else { - updateNeeded = true - } - if !updateNeeded { - // All profiles are already in the database, no need to update. - return nil - } - + // If this call is made, we assume the update must be done -- a new profile was added or existing one modified. updatedDB = true baseStmt := fmt.Sprintf(` INSERT INTO host_mdm_apple_profiles ( @@ -2224,6 +2217,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( profile_identifier, profile_name, checksum, + secrets_updated_at, operation_type, status, command_uuid, @@ -2235,6 +2229,7 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( status = VALUES(status), command_uuid = VALUES(command_uuid), checksum = VALUES(checksum), + secrets_updated_at = VALUES(secrets_updated_at), detail = VALUES(detail) `, strings.TrimSuffix(valuePart, ",")) @@ -2271,14 +2266,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( HostUUID: p.HostUUID, HostPlatform: p.HostPlatform, Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, Status: pp.Status, OperationType: pp.OperationType, Detail: pp.Detail, CommandUUID: pp.CommandUUID, } - pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, + pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt, pp.OperationType, pp.Status, pp.CommandUUID, pp.Detail) - psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),") + psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),") batchCount++ if batchCount >= batchSize { @@ -2298,14 +2294,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( HostUUID: p.HostUUID, HostPlatform: p.HostPlatform, Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, OperationType: fleet.MDMOperationTypeInstall, Status: nil, CommandUUID: "", Detail: "", } - pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, + pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt, fleet.MDMOperationTypeInstall, nil, "", "") - psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),") + psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),") batchCount++ if batchCount >= batchSize { @@ -2334,14 +2331,15 @@ func (ds *Datastore) bulkSetPendingMDMAppleHostProfilesDB( HostUUID: p.HostUUID, HostPlatform: p.HostPlatform, Checksum: p.Checksum, + SecretsUpdatedAt: p.SecretsUpdatedAt, OperationType: fleet.MDMOperationTypeRemove, Status: nil, CommandUUID: "", Detail: "", } - pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, + pargs = append(pargs, p.ProfileUUID, p.HostUUID, p.ProfileIdentifier, p.ProfileName, p.Checksum, p.SecretsUpdatedAt, fleet.MDMOperationTypeRemove, nil, "", "") - psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?),") + psb.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?),") batchCount++ if batchCount >= batchSize { @@ -2404,6 +2402,7 @@ func generateDesiredStateQuery(entityType string) string { mae.identifier as ${entityIdentifierColumn}, mae.name as ${entityNameColumn}, mae.checksum as checksum, + mae.secrets_updated_at as secrets_updated_at, 0 as ${countEntityLabelsColumn}, 0 as count_non_broken_labels, 0 as count_host_labels, @@ -2437,6 +2436,7 @@ func generateDesiredStateQuery(entityType string) string { mae.identifier as ${entityIdentifierColumn}, mae.name as ${entityNameColumn}, mae.checksum as checksum, + mae.secrets_updated_at as secrets_updated_at, COUNT(*) as ${countEntityLabelsColumn}, COUNT(mel.label_id) as count_non_broken_labels, COUNT(lm.label_id) as count_host_labels, @@ -2457,7 +2457,7 @@ func generateDesiredStateQuery(entityType string) string { ne.type = 'Device' AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at HAVING ${countEntityLabelsColumn} > 0 AND count_host_labels = ${countEntityLabelsColumn} @@ -2475,6 +2475,7 @@ func generateDesiredStateQuery(entityType string) string { mae.identifier as ${entityIdentifierColumn}, mae.name as ${entityNameColumn}, mae.checksum as checksum, + mae.secrets_updated_at as secrets_updated_at, COUNT(*) as ${countEntityLabelsColumn}, COUNT(mel.label_id) as count_non_broken_labels, COUNT(lm.label_id) as count_host_labels, @@ -2499,7 +2500,7 @@ func generateDesiredStateQuery(entityType string) string { ne.type = 'Device' AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at HAVING -- considers only the profiles with labels, without any broken label, with results reported after all labels were created and with the host not in any label ${countEntityLabelsColumn} > 0 AND ${countEntityLabelsColumn} = count_non_broken_labels AND ${countEntityLabelsColumn} = count_host_updated_after_labels AND count_host_labels = 0 @@ -2516,6 +2517,7 @@ func generateDesiredStateQuery(entityType string) string { mae.identifier as ${entityIdentifierColumn}, mae.name as ${entityNameColumn}, mae.checksum as checksum, + mae.secrets_updated_at as secrets_updated_at, COUNT(*) as ${countEntityLabelsColumn}, COUNT(mel.label_id) as count_non_broken_labels, COUNT(lm.label_id) as count_host_labels, @@ -2536,7 +2538,7 @@ func generateDesiredStateQuery(entityType string) string { ne.type = 'Device' AND ( %s ) GROUP BY - mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum + mae.${entityUUIDColumn}, h.uuid, h.platform, mae.identifier, mae.name, mae.checksum, mae.secrets_updated_at HAVING ${countEntityLabelsColumn} > 0 AND count_host_labels >= 1 `, func(s string) string { return dynamicNames[s] }) @@ -2590,7 +2592,7 @@ func generateEntitiesToInstallQuery(entityType string) string { ON hmae.${entityUUIDColumn} = ds.${entityUUIDColumn} AND hmae.host_uuid = ds.host_uuid WHERE -- entity has been updated - ( hmae.checksum != ds.checksum ) OR + ( hmae.checksum != ds.checksum ) OR ( hmae.secrets_updated_at IS NOT NULL AND hmae.secrets_updated_at < ds.secrets_updated_at ) OR -- entity in A but not in B ( hmae.${entityUUIDColumn} IS NULL AND hmae.host_uuid IS NULL ) OR -- entities in A and B but with operation type "remove" @@ -2661,7 +2663,8 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee ds.host_platform, ds.profile_identifier, ds.profile_name, - ds.checksum + ds.checksum, + ds.secrets_updated_at FROM %s `, generateEntitiesToInstallQuery("profile")) var profiles []*fleet.MDMAppleProfilePayload @@ -2670,6 +2673,7 @@ func (ds *Datastore) ListMDMAppleProfilesToInstall(ctx context.Context) ([]*flee } func (ds *Datastore) ListMDMAppleProfilesToRemove(ctx context.Context) ([]*fleet.MDMAppleProfilePayload, error) { + // Note: we don't include SecretsUpdatedAt timestamp because it should not be needed for profile removal query := fmt.Sprintf(` SELECT hmae.profile_uuid, @@ -2718,6 +2722,8 @@ func (ds *Datastore) GetMDMAppleProfilesContents(ctx context.Context, uuids []st return results, nil } +// BulkUpsertMDMAppleHostProfiles is used to update the status of profile delivery to hosts. +// It is not intended to update the contents of the profiles themselves. Hence, the secrets_updated_at timestamp is not updated here. func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload []*fleet.MDMAppleBulkUpsertHostProfilePayload) error { if len(payload) == 0 { return nil @@ -2742,6 +2748,7 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload operation_type = VALUES(operation_type), detail = VALUES(detail), checksum = VALUES(checksum), + secrets_updated_at = VALUES(secrets_updated_at), profile_identifier = VALUES(profile_identifier), profile_name = VALUES(profile_name), command_uuid = VALUES(command_uuid)`, @@ -2761,7 +2768,7 @@ func (ds *Datastore) BulkUpsertMDMAppleHostProfiles(ctx context.Context, payload } generateValueArgs := func(p *fleet.MDMAppleBulkUpsertHostProfilePayload) (string, []any) { - valuePart := "(?, ?, ?, ?, ?, ?, ?, ?, ?)," + valuePart := "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)," args := []any{p.ProfileUUID, p.ProfileIdentifier, p.ProfileName, p.HostUUID, p.Status, p.OperationType, p.Detail, p.CommandUUID, p.Checksum} return valuePart, args } @@ -4652,7 +4659,7 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { const stmt = ` SELECT - COALESCE(MD5((count(0) + GROUP_CONCAT(HEX(mad.checksum) + COALESCE(MD5((count(0) + GROUP_CONCAT(CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, '')) ORDER BY mad.uploaded_at DESC separator ''))), '') AS checksum, COALESCE(MAX(mad.created_at), NOW()) AS latest_created_timestamp @@ -4681,7 +4688,7 @@ WHERE func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { const stmt = ` SELECT - HEX(mad.checksum) as checksum, + CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, '')) as checksum, mad.identifier FROM host_mdm_apple_declarations hmad @@ -4704,7 +4711,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identi // declarations are removed, but the join would provide an extra layer of safety. const stmt = ` SELECT - mad.raw_json, HEX(mad.checksum) as checksum + mad.raw_json, CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, '')) as checksum FROM host_mdm_apple_declarations hmad JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid @@ -4792,58 +4799,20 @@ func mdmAppleBatchSetPendingHostDeclarationsDB( ) (updatedDB bool, err error) { baseStmt := ` INSERT INTO host_mdm_apple_declarations - (host_uuid, status, operation_type, checksum, declaration_uuid, declaration_identifier, declaration_name) + (host_uuid, status, operation_type, checksum, secrets_updated_at, declaration_uuid, declaration_identifier, declaration_name) VALUES %s ON DUPLICATE KEY UPDATE status = VALUES(status), operation_type = VALUES(operation_type), - checksum = VALUES(checksum) + checksum = VALUES(checksum), + secrets_updated_at = VALUES(secrets_updated_at) ` profilesToInsert := make(map[string]*fleet.MDMAppleHostDeclaration) executeUpsertBatch := func(valuePart string, args []any) error { - // Check if the update needs to be done at all. - selectStmt := fmt.Sprintf(` - SELECT - host_uuid, - declaration_uuid, - status, - COALESCE(operation_type, '') AS operation_type, - COALESCE(detail, '') AS detail, - checksum, - declaration_uuid, - declaration_identifier, - declaration_name - FROM host_mdm_apple_declarations WHERE (host_uuid, declaration_uuid) IN (%s)`, - strings.TrimSuffix(strings.Repeat("(?,?),", len(profilesToInsert)), ",")) - var selectArgs []any - for _, p := range profilesToInsert { - selectArgs = append(selectArgs, p.HostUUID, p.DeclarationUUID) - } - var existingProfiles []fleet.MDMAppleHostDeclaration - if err := sqlx.SelectContext(ctx, tx, &existingProfiles, selectStmt, selectArgs...); err != nil { - return ctxerr.Wrap(ctx, err, "bulk set pending declarations select existing") - } - var updateNeeded bool - if len(existingProfiles) == len(profilesToInsert) { - for _, exist := range existingProfiles { - insert, ok := profilesToInsert[fmt.Sprintf("%s\n%s", exist.HostUUID, exist.DeclarationUUID)] - if !ok || !exist.Equal(*insert) { - updateNeeded = true - break - } - } - } else { - updateNeeded = true - } - clear(profilesToInsert) - if !updateNeeded { - // All profiles are already in the database, no need to update. - return nil - } - + // If we're here, we must have a batch of declarations to insert/update. updatedDB = true _, err := tx.ExecContext( ctx, @@ -4855,17 +4824,18 @@ func mdmAppleBatchSetPendingHostDeclarationsDB( generateValueArgs := func(d *fleet.MDMAppleHostDeclaration) (string, []any) { profilesToInsert[fmt.Sprintf("%s\n%s", d.HostUUID, d.DeclarationUUID)] = &fleet.MDMAppleHostDeclaration{ - HostUUID: d.HostUUID, - DeclarationUUID: d.DeclarationUUID, - Name: d.Name, - Identifier: d.Identifier, - Status: status, - OperationType: d.OperationType, - Detail: d.Detail, - Checksum: d.Checksum, - } - valuePart := "(?, ?, ?, ?, ?, ?, ?)," - args := []any{d.HostUUID, status, d.OperationType, d.Checksum, d.DeclarationUUID, d.Identifier, d.Name} + HostUUID: d.HostUUID, + DeclarationUUID: d.DeclarationUUID, + Name: d.Name, + Identifier: d.Identifier, + Status: status, + OperationType: d.OperationType, + Detail: d.Detail, + Checksum: d.Checksum, + SecretsUpdatedAt: d.SecretsUpdatedAt, + } + valuePart := "(?, ?, ?, ?, ?, ?, ?, ?)," + args := []any{d.HostUUID, status, d.OperationType, d.Checksum, d.SecretsUpdatedAt, d.DeclarationUUID, d.Identifier, d.Name} return valuePart, args } @@ -4885,6 +4855,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC ds.host_uuid, 'install' as operation_type, ds.checksum, + ds.secrets_updated_at, ds.declaration_uuid, ds.declaration_identifier, ds.declaration_name @@ -4897,6 +4868,7 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC hmae.host_uuid, 'remove' as operation_type, hmae.checksum, + COALESCE(hmae.secrets_updated_at, NOW(6)) as secrets_updated_at, hmae.declaration_uuid, hmae.declaration_identifier, hmae.declaration_name @@ -4915,6 +4887,8 @@ func mdmAppleGetHostsWithChangedDeclarationsDB(ctx context.Context, tx sqlx.ExtC return decls, nil } +// MDMAppleStoreDDMStatusReport updates the status of the host's declarations. +// Note: SecretsUpdatedAt timestamp is not used/updated in this method. func (ds *Datastore) MDMAppleStoreDDMStatusReport(ctx context.Context, hostUUID string, updates []*fleet.MDMAppleHostDeclaration) error { getHostDeclarationsStmt := ` SELECT host_uuid, status, operation_type, HEX(checksum) as checksum, declaration_uuid, declaration_identifier, declaration_name diff --git a/server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go b/server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go new file mode 100644 index 000000000000..b03742b5e404 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go @@ -0,0 +1,51 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20241223115925, Down_20241223115925) +} + +func Up_20241223115925(tx *sql.Tx) error { + // secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed + _, err := tx.Exec(`ALTER TABLE mdm_apple_configuration_profiles + ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`) + if err != nil { + return fmt.Errorf("failed to alter mdm_apple_configuration_profiles table: %w", err) + } + + // Add secrets_updated_at column to host_mdm_apple_profiles + _, err = tx.Exec(`ALTER TABLE host_mdm_apple_profiles + ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`) + if err != nil { + return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_profiles table: %w", err) + } + + // secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed + _, err = tx.Exec(`ALTER TABLE mdm_apple_declarations + ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`) + if err != nil { + return fmt.Errorf("failed to alter mdm_apple_declarations table: %w", err) + } + + // Add secrets_updated_at column to host_mdm_apple_declarations + _, err = tx.Exec(`ALTER TABLE host_mdm_apple_declarations + -- defaulting to NULL for backward compatibility with existing declarations + ADD COLUMN secrets_updated_at TIMESTAMP(6) NULL`) + if err != nil { + return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_declarations table: %w", err) + } + + return nil +} + +func Down_20241223115925(_ *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index b9ca48df03aa..2a8f9359869f 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -423,12 +423,13 @@ CREATE TABLE `host_mdm_apple_declarations` ( `declaration_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `declaration_identifier` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `declaration_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `secrets_updated_at` timestamp(6) NULL DEFAULT NULL, PRIMARY KEY (`host_uuid`,`declaration_uuid`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), CONSTRAINT `host_mdm_apple_declarations_ibfk_1` FOREIGN KEY (`status`) REFERENCES `mdm_delivery_status` (`status`) ON UPDATE CASCADE, CONSTRAINT `host_mdm_apple_declarations_ibfk_2` FOREIGN KEY (`operation_type`) REFERENCES `mdm_operation_types` (`operation_type`) ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; @@ -445,6 +446,7 @@ CREATE TABLE `host_mdm_apple_profiles` ( `profile_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', `created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), `updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), + `secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (`host_uuid`,`profile_uuid`), KEY `status` (`status`), KEY `operation_type` (`operation_type`), @@ -848,10 +850,11 @@ CREATE TABLE `mdm_apple_configuration_profiles` ( `identifier` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, `mobileconfig` mediumblob NOT NULL, - `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `uploaded_at` timestamp NULL DEFAULT NULL, + `created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `uploaded_at` timestamp(6) NULL DEFAULT NULL, `checksum` binary(16) NOT NULL, `profile_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', + `secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (`profile_uuid`), UNIQUE KEY `idx_mdm_apple_config_prof_team_identifier` (`team_id`,`identifier`), UNIQUE KEY `idx_mdm_apple_config_prof_team_name` (`team_id`,`name`), @@ -878,9 +881,10 @@ CREATE TABLE `mdm_apple_declarations` ( `name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `raw_json` mediumtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `checksum` binary(16) NOT NULL, - `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, - `uploaded_at` timestamp NULL DEFAULT NULL, + `created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), + `uploaded_at` timestamp(6) NULL DEFAULT NULL, `auto_increment` bigint NOT NULL AUTO_INCREMENT, + `secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6), PRIMARY KEY (`declaration_uuid`), UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`), UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`), @@ -1105,9 +1109,9 @@ CREATE TABLE `migration_status_tables` ( `is_applied` tinyint(1) NOT NULL, `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=341 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=342 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241110152839,1,'2020-01-01 01:01:01'),(328,20241110152840,1,'2020-01-01 01:01:01'),(329,20241110152841,1,'2020-01-01 01:01:01'),(330,20241116233322,1,'2020-01-01 01:01:01'),(331,20241122171434,1,'2020-01-01 01:01:01'),(332,20241125150614,1,'2020-01-01 01:01:01'),(333,20241203125346,1,'2020-01-01 01:01:01'),(334,20241203130032,1,'2020-01-01 01:01:01'),(335,20241205122800,1,'2020-01-01 01:01:01'),(336,20241209164540,1,'2020-01-01 01:01:01'),(337,20241210140021,1,'2020-01-01 01:01:01'),(338,20241219180042,1,'2020-01-01 01:01:01'),(339,20241220100000,1,'2020-01-01 01:01:01'),(340,20241220114903,1,'2020-01-01 01:01:01'),(341,20241223115925,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/datastore/mysql/secret_variables.go b/server/datastore/mysql/secret_variables.go index 19245dbc76f8..bb6f6561afce 100644 --- a/server/datastore/mysql/secret_variables.go +++ b/server/datastore/mysql/secret_variables.go @@ -2,8 +2,11 @@ package mysql import ( "context" + "database/sql" + "errors" "fmt" "strings" + "time" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" @@ -15,24 +18,69 @@ func (ds *Datastore) UpsertSecretVariables(ctx context.Context, secretVariables return nil } - values := strings.TrimSuffix(strings.Repeat("(?,?),", len(secretVariables)), ",") + // The secret variables should rarely change, so we do not use a transaction here. + // When we encrypt a secret variable, it is salted, so the encrypted data is different each time. + // In order to keep the updated_at timestamp correct, we need to compare the encrypted value + // with the existing value in the database. If the values are the same, we do not update the row. - stmt := fmt.Sprintf(` - INSERT INTO secret_variables (name, value) - VALUES %s - ON DUPLICATE KEY UPDATE value = VALUES(value)`, values) - - args := make([]interface{}, 0, len(secretVariables)*2) + var names []string for _, secretVariable := range secretVariables { - valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey) - if err != nil { - return ctxerr.Wrap(ctx, err, "encrypt secret value with server private key") + names = append(names, secretVariable.Name) + } + existingVariables, err := ds.GetSecretVariables(ctx, names) + if err != nil { + return ctxerr.Wrap(ctx, err, "get existing secret variables") + } + existingVariableMap := make(map[string]string, len(existingVariables)) + for _, existingVariable := range existingVariables { + existingVariableMap[existingVariable.Name] = existingVariable.Value + } + var variablesToInsert []fleet.SecretVariable + var variablesToUpdate []fleet.SecretVariable + for _, secretVariable := range secretVariables { + existingValue, ok := existingVariableMap[secretVariable.Name] + switch { + case !ok: + variablesToInsert = append(variablesToInsert, secretVariable) + case existingValue != secretVariable.Value: + variablesToUpdate = append(variablesToUpdate, secretVariable) + default: + // No change -- the variable value is the same } - args = append(args, secretVariable.Name, valueEncrypted) } - if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { - return ctxerr.Wrap(ctx, err, "upsert secret variables") + if len(variablesToInsert) > 0 { + values := strings.TrimSuffix(strings.Repeat("(?,?),", len(variablesToInsert)), ",") + stmt := fmt.Sprintf(` + INSERT INTO secret_variables (name, value) + VALUES %s`, values) + args := make([]interface{}, 0, len(variablesToInsert)*2) + for _, secretVariable := range variablesToInsert { + valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey) + if err != nil { + return ctxerr.Wrap(ctx, err, "encrypt secret value for insert with server private key") + } + args = append(args, secretVariable.Name, valueEncrypted) + } + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, args...); err != nil { + return ctxerr.Wrap(ctx, err, "insert secret variables") + } + } + + if len(variablesToUpdate) > 0 { + stmt := ` + UPDATE secret_variables + SET value = ? + WHERE name = ?` + for _, secretVariable := range variablesToUpdate { + valueEncrypted, err := encrypt([]byte(secretVariable.Value), ds.serverPrivateKey) + if err != nil { + return ctxerr.Wrap(ctx, err, "encrypt secret value for update with server private key") + } + if _, err := ds.writer(ctx).ExecContext(ctx, stmt, valueEncrypted, secretVariable.Name); err != nil { + return ctxerr.Wrap(ctx, err, "update secret variables") + } + } } return nil @@ -69,6 +117,31 @@ func (ds *Datastore) GetSecretVariables(ctx context.Context, names []string) ([] return secretVariables, nil } +func (ds *Datastore) secretVariablesUpdated(ctx context.Context, q sqlx.QueryerContext, names []string, timestamp time.Time) (bool, error) { + if len(names) == 0 { + return false, nil + } + + stmt, args, err := sqlx.In(` + SELECT 1 + FROM secret_variables + WHERE name IN (?) AND updated_at > ?`, names, timestamp) + if err != nil { + return false, ctxerr.Wrap(ctx, err, "build secret variables query") + } + + var updated bool + err = sqlx.GetContext(ctx, q, &updated, stmt, args...) + switch { + case errors.Is(err, sql.ErrNoRows): + return false, nil + case err != nil: + return false, ctxerr.Wrap(ctx, err, "get secret variables") + default: + return updated, nil + } +} + func (ds *Datastore) ExpandEmbeddedSecrets(ctx context.Context, document string) (string, error) { embeddedSecrets := fleet.ContainsPrefixVars(document, fleet.ServerSecretPrefix) if len(embeddedSecrets) == 0 { diff --git a/server/datastore/mysql/secret_variables_test.go b/server/datastore/mysql/secret_variables_test.go index 8936442b58f0..e2afe6c136c5 100644 --- a/server/datastore/mysql/secret_variables_test.go +++ b/server/datastore/mysql/secret_variables_test.go @@ -3,8 +3,10 @@ package mysql import ( "context" "testing" + "time" "github.com/fleetdm/fleet/v4/server/fleet" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -59,18 +61,35 @@ func testUpsertSecretVariables(t *testing.T, ds *Datastore) { assert.Equal(t, secretMap[result.Name], result.Value) } - // Update a secret + // Update a secret and insert a new one secretMap["test2"] = "newTestValue2" + secretMap["test4"] = "testValue4" err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{ {Name: "test2", Value: secretMap["test2"]}, + {Name: "test4", Value: secretMap["test4"]}, }) assert.NoError(t, err) - results, err = ds.GetSecretVariables(ctx, []string{"test2"}) + results, err = ds.GetSecretVariables(ctx, []string{"test2", "test4"}) assert.NoError(t, err) - require.Len(t, results, 1) - assert.Equal(t, "test2", results[0].Name) - assert.Equal(t, secretMap[results[0].Name], results[0].Value) + require.Len(t, results, 2) + for _, result := range results { + assert.Equal(t, secretMap[result.Name], result.Value) + } + // Make sure updated_at timestamp does not change when we update a secret with the same value + var myTime time.Time + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &myTime, "SELECT updated_at FROM secret_variables WHERE name = ?", "test1") + }) + err = ds.UpsertSecretVariables(ctx, []fleet.SecretVariable{ + {Name: "test1", Value: secretMap["test1"]}, + }) + assert.NoError(t, err) + var newTime time.Time + ExecAdhocSQL(t, ds, func(q sqlx.ExtContext) error { + return sqlx.GetContext(ctx, q, &newTime, "SELECT updated_at FROM secret_variables WHERE name = ?", "test1") + }) + assert.Equal(t, myTime, newTime) } func testValidateEmbeddedSecrets(t *testing.T, ds *Datastore) { diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 3cf0378e9114..ffe24e8d29f1 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -1,7 +1,6 @@ package fleet import ( - "bytes" "context" "crypto/md5" // nolint: gosec "encoding/hex" @@ -205,6 +204,7 @@ type MDMAppleConfigProfile struct { LabelsExcludeAny []ConfigurationProfileLabel `db:"-" json:"labels_exclude_any,omitempty"` CreatedAt time.Time `db:"created_at" json:"created_at"` UploadedAt time.Time `db:"uploaded_at" json:"updated_at"` // NOTE: JSON field is still `updated_at` for historical reasons, would be an API breaking change + SecretsUpdatedAt time.Time `db:"secrets_updated_at" json:"-"` } // MDMProfilesUpdates flags updates that were done during batch processing of profiles. @@ -321,6 +321,7 @@ type MDMAppleProfilePayload struct { HostUUID string `db:"host_uuid"` HostPlatform string `db:"host_platform"` Checksum []byte `db:"checksum"` + SecretsUpdatedAt time.Time `db:"secrets_updated_at"` Status *MDMDeliveryStatus `db:"status" json:"status"` OperationType MDMOperationType `db:"operation_type"` Detail string `db:"detail"` @@ -333,20 +334,6 @@ func (p *MDMAppleProfilePayload) DidNotInstallOnHost() bool { return p.Status != nil && (*p.Status == MDMDeliveryFailed || *p.Status == MDMDeliveryPending) && p.OperationType == MDMOperationTypeInstall } -func (p MDMAppleProfilePayload) Equal(other MDMAppleProfilePayload) bool { - statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status - return p.ProfileUUID == other.ProfileUUID && - p.ProfileIdentifier == other.ProfileIdentifier && - p.ProfileName == other.ProfileName && - p.HostUUID == other.HostUUID && - p.HostPlatform == other.HostPlatform && - bytes.Equal(p.Checksum, other.Checksum) && - statusEqual && - p.OperationType == other.OperationType && - p.Detail == other.Detail && - p.CommandUUID == other.CommandUUID -} - type MDMAppleBulkUpsertHostProfilePayload struct { ProfileUUID string ProfileIdentifier string @@ -697,18 +684,9 @@ type MDMAppleHostDeclaration struct { // Checksum contains the MD5 checksum of the declaration JSON uploaded // by the IT admin. Fleet uses this value as the ServerToken. Checksum string `db:"checksum" json:"-"` -} -func (p MDMAppleHostDeclaration) Equal(other MDMAppleHostDeclaration) bool { - statusEqual := p.Status == nil && other.Status == nil || p.Status != nil && other.Status != nil && *p.Status == *other.Status - return statusEqual && - p.HostUUID == other.HostUUID && - p.DeclarationUUID == other.DeclarationUUID && - p.Name == other.Name && - p.Identifier == other.Identifier && - p.OperationType == other.OperationType && - p.Detail == other.Detail && - p.Checksum == other.Checksum + // SecretsUpdatedAt is the timestamp when the secrets were last updated or when this declaration was uploaded. + SecretsUpdatedAt time.Time `db:"secrets_updated_at" json:"-"` } func NewMDMAppleDeclaration(raw []byte, teamID *uint, name string, declType, ident string) *MDMAppleDeclaration { diff --git a/server/fleet/apple_mdm_test.go b/server/fleet/apple_mdm_test.go index 924e51aa874f..cf18288e5e82 100644 --- a/server/fleet/apple_mdm_test.go +++ b/server/fleet/apple_mdm_test.go @@ -419,147 +419,6 @@ func TestMDMProfileIsWithinGracePeriod(t *testing.T) { } } -func TestMDMAppleHostDeclarationEqual(t *testing.T) { - t.Parallel() - - // This test is intended to ensure that the Equal method on MDMAppleHostDeclaration is updated when new fields are added. - // The Equal method is used to identify whether database update is needed. - - items := [...]MDMAppleHostDeclaration{{}, {}} - - numberOfFields := 0 - for i := 0; i < len(items); i++ { - rValue := reflect.ValueOf(&items[i]).Elem() - numberOfFields = rValue.NumField() - for j := 0; j < numberOfFields; j++ { - field := rValue.Field(j) - switch field.Kind() { - case reflect.String: - valueToSet := fmt.Sprintf("test %d", i) - field.SetString(valueToSet) - case reflect.Int: - field.SetInt(int64(i)) - case reflect.Bool: - field.SetBool(i%2 == 0) - case reflect.Pointer: - field.Set(reflect.New(field.Type().Elem())) - default: - t.Fatalf("unhandled field type %s", field.Kind()) - } - } - } - - status0 := MDMDeliveryStatus("status") - status1 := MDMDeliveryStatus("status") - items[0].Status = &status0 - assert.False(t, items[0].Equal(items[1])) - - // Set known fields to be equal - fieldsInEqualMethod := 0 - items[1].HostUUID = items[0].HostUUID - fieldsInEqualMethod++ - items[1].DeclarationUUID = items[0].DeclarationUUID - fieldsInEqualMethod++ - items[1].Name = items[0].Name - fieldsInEqualMethod++ - items[1].Identifier = items[0].Identifier - fieldsInEqualMethod++ - items[1].OperationType = items[0].OperationType - fieldsInEqualMethod++ - items[1].Detail = items[0].Detail - fieldsInEqualMethod++ - items[1].Checksum = items[0].Checksum - fieldsInEqualMethod++ - items[1].Status = &status1 - fieldsInEqualMethod++ - assert.Equal(t, fieldsInEqualMethod, numberOfFields, "MDMAppleHostDeclaration.Equal needs to be updated for new/updated field(s)") - assert.True(t, items[0].Equal(items[1])) - - // Set pointers to nil - items[0].Status = nil - items[1].Status = nil - assert.True(t, items[0].Equal(items[1])) -} - -func TestMDMAppleProfilePayloadEqual(t *testing.T) { - t.Parallel() - - // This test is intended to ensure that the Equal method on MDMAppleProfilePayload is updated when new fields are added. - // The Equal method is used to identify whether database update is needed. - - items := [...]MDMAppleProfilePayload{{}, {}} - - numberOfFields := 0 - for i := 0; i < len(items); i++ { - rValue := reflect.ValueOf(&items[i]).Elem() - numberOfFields = rValue.NumField() - for j := 0; j < numberOfFields; j++ { - field := rValue.Field(j) - switch field.Kind() { - case reflect.String: - valueToSet := fmt.Sprintf("test %d", i) - field.SetString(valueToSet) - case reflect.Int: - field.SetInt(int64(i)) - case reflect.Bool: - field.SetBool(i%2 == 0) - case reflect.Pointer: - field.Set(reflect.New(field.Type().Elem())) - case reflect.Slice: - switch field.Type().Elem().Kind() { - case reflect.Uint8: - valueToSet := []byte("test") - field.Set(reflect.ValueOf(valueToSet)) - default: - t.Fatalf("unhandled slice type %s", field.Type().Elem().Kind()) - } - default: - t.Fatalf("unhandled field type %s", field.Kind()) - } - } - } - - status0 := MDMDeliveryStatus("status") - status1 := MDMDeliveryStatus("status") - items[0].Status = &status0 - checksum0 := []byte("checksum") - checksum1 := []byte("checksum") - items[0].Checksum = checksum0 - assert.False(t, items[0].Equal(items[1])) - - // Set known fields to be equal - fieldsInEqualMethod := 0 - items[1].ProfileUUID = items[0].ProfileUUID - fieldsInEqualMethod++ - items[1].ProfileIdentifier = items[0].ProfileIdentifier - fieldsInEqualMethod++ - items[1].ProfileName = items[0].ProfileName - fieldsInEqualMethod++ - items[1].HostUUID = items[0].HostUUID - fieldsInEqualMethod++ - items[1].HostPlatform = items[0].HostPlatform - fieldsInEqualMethod++ - items[1].Checksum = checksum1 - fieldsInEqualMethod++ - items[1].Status = &status1 - fieldsInEqualMethod++ - items[1].OperationType = items[0].OperationType - fieldsInEqualMethod++ - items[1].Detail = items[0].Detail - fieldsInEqualMethod++ - items[1].CommandUUID = items[0].CommandUUID - fieldsInEqualMethod++ - assert.Equal(t, fieldsInEqualMethod, numberOfFields, "MDMAppleProfilePayload.Equal needs to be updated for new/updated field(s)") - assert.True(t, items[0].Equal(items[1])) - - // Set pointers and slices to nil - items[0].Status = nil - items[1].Status = nil - items[0].Checksum = nil - items[1].Checksum = nil - assert.True(t, items[0].Equal(items[1])) -} - func TestConfigurationProfileLabelEqual(t *testing.T) { t.Parallel()