Skip to content

Commit 91ff746

Browse files
getvictorgetvictor
committed
Work in progress.
1 parent aee52f1 commit 91ff746

File tree

10 files changed

+119
-37
lines changed

10 files changed

+119
-37
lines changed

‎server/datastore/mysql/apple_mdm.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1737,7 +1737,7 @@ ON DUPLICATE KEY UPDATE
17371737
// nolint:gosec // ignore G101: Potential hardcoded credentials
17381738
const secretsUpdatedInProfile = `
17391739
UPDATE mdm_apple_configuration_profiles
1740-
SET secrets_updated_at = CURRENT_TIMESTAMP(6)
1740+
SET secrets_updated_at = NOW(6)
17411741
WHERE team_id = ? AND identifier = ?`
17421742

17431743
// use a profile team id of 0 if no-team
@@ -4659,7 +4659,7 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont
46594659
func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) {
46604660
const stmt = `
46614661
SELECT
4662-
COALESCE(MD5((count(0) + GROUP_CONCAT(CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, ''))
4662+
COALESCE(MD5((count(0) + GROUP_CONCAT(hmad.token
46634663
ORDER BY
46644664
mad.uploaded_at DESC separator ''))), '') AS checksum,
46654665
COALESCE(MAX(mad.created_at), NOW()) AS latest_created_timestamp
@@ -4688,7 +4688,7 @@ WHERE
46884688
func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) {
46894689
const stmt = `
46904690
SELECT
4691-
CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, '')) as checksum,
4691+
token,
46924692
mad.identifier
46934693
FROM
46944694
host_mdm_apple_declarations hmad
@@ -4711,7 +4711,7 @@ func (ds *Datastore) MDMAppleDDMDeclarationsResponse(ctx context.Context, identi
47114711
// declarations are removed, but the join would provide an extra layer of safety.
47124712
const stmt = `
47134713
SELECT
4714-
mad.raw_json, CONCAT(HEX(mad.checksum), COALESCE(hmad.secrets_updated_at, '')) as checksum
4714+
mad.raw_json, token
47154715
FROM
47164716
host_mdm_apple_declarations hmad
47174717
JOIN mdm_apple_declarations mad ON hmad.declaration_uuid = mad.declaration_uuid

‎server/datastore/mysql/apple_mdm_test.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1120,7 +1120,7 @@ func expectAppleDeclarations(
11201120
require.NotEmpty(t, gotD.DeclarationUUID)
11211121
require.True(t, strings.HasPrefix(gotD.DeclarationUUID, fleet.MDMAppleDeclarationUUIDPrefix))
11221122
gotD.DeclarationUUID = ""
1123-
gotD.Checksum = "" // don't care about md5checksum here
1123+
gotD.Token = "" // don't care about md5checksum here
11241124

11251125
gotD.CreatedAt = time.Time{}
11261126

@@ -1400,6 +1400,7 @@ func testMDMAppleProfileManagement(t *testing.T, ds *Datastore) {
14001400
for _, p := range got {
14011401
require.NotEmpty(t, p.Checksum)
14021402
p.Checksum = nil
1403+
p.SecretsUpdatedAt = nil
14031404
}
14041405
require.ElementsMatch(t, want, got)
14051406
}
@@ -7337,7 +7338,10 @@ func testMDMAppleProfileLabels(t *testing.T, ds *Datastore) {
73377338
matchProfiles := func(want, got []*fleet.MDMAppleProfilePayload) {
73387339
// match only the fields we care about
73397340
for _, p := range got {
7341+
assert.NotEmpty(t, p.Checksum)
73407342
p.Checksum = nil
7343+
assert.NotZero(t, p.SecretsUpdatedAt)
7344+
p.SecretsUpdatedAt = nil
73417345
}
73427346
require.ElementsMatch(t, want, got)
73437347
}

‎server/datastore/mysql/mdm.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,7 @@ OR
546546
//
547547
// Note(victor): Why is the status being set to nil? Shouldn't it be set to pending?
548548
// Or at least pending for install and nil for remove profiles. Please update this comment if you know.
549+
// This method is called bulkSetPendingMDMHostProfilesDB, so it is confusing that the status is NOT explicitly set to pending.
549550
_, updates.AppleDeclaration, err = mdmAppleBatchSetHostDeclarationStateDB(ctx, tx, batchSize, nil)
550551
if err != nil {
551552
return updates, ctxerr.Wrap(ctx, err, "bulk set pending apple declarations")

‎server/datastore/mysql/migrations/tables/20241223115925_AddSecretsUpdatedAt.go

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,12 @@ func init() {
1010
}
1111

1212
func Up_20241223115925(tx *sql.Tx) error {
13+
// Using DATETIME instead of TIMESTAMP for secrets_updated_at to avoid future Y2K38 issues,
14+
// since this date is used to detect if profile needs to be updated.
15+
1316
// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
1417
_, err := tx.Exec(`ALTER TABLE mdm_apple_configuration_profiles
15-
ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
18+
ADD COLUMN secrets_updated_at DATETIME(6) NULL,
1619
MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
1720
MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`)
1821
if err != nil {
@@ -21,24 +24,33 @@ func Up_20241223115925(tx *sql.Tx) error {
2124

2225
// Add secrets_updated_at column to host_mdm_apple_profiles
2326
_, err = tx.Exec(`ALTER TABLE host_mdm_apple_profiles
24-
ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6)`)
27+
ADD COLUMN secrets_updated_at DATETIME(6) NULL`)
2528
if err != nil {
2629
return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_profiles table: %w", err)
2730
}
2831

2932
// secrets_updated_at are updated when profile contents have not changed but secret variables in the profile have changed
3033
_, err = tx.Exec(`ALTER TABLE mdm_apple_declarations
31-
ADD COLUMN secrets_updated_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
34+
ADD COLUMN secrets_updated_at DATETIME(6) NULL,
3235
MODIFY COLUMN created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
33-
MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL`)
36+
MODIFY COLUMN uploaded_at TIMESTAMP(6) NULL DEFAULT NULL,
37+
-- token is used to identify if declaration needs to be re-applied
38+
-- using HEX for consistency and slight compression
39+
-- mdm_apple_declarations.token formula must match host_mdm_apple_declarations.token formula
40+
ADD COLUMN token VARCHAR(63) COLLATE utf8mb4_unicode_ci GENERATED ALWAYS AS
41+
(CONCAT(HEX(checksum), COALESCE(CONCAT('-', HEX(TO_SECONDS(secrets_updated_at)), '-', HEX(MICROSECOND(secrets_updated_at)))), '')) VIRTUAL NULL`)
3442
if err != nil {
3543
return fmt.Errorf("failed to alter mdm_apple_declarations table: %w", err)
3644
}
3745

3846
// Add secrets_updated_at column to host_mdm_apple_declarations
3947
_, err = tx.Exec(`ALTER TABLE host_mdm_apple_declarations
4048
-- defaulting to NULL for backward compatibility with existing declarations
41-
ADD COLUMN secrets_updated_at TIMESTAMP(6) NULL`)
49+
ADD COLUMN secrets_updated_at DATETIME(6) NULL,
50+
-- token is used to identify if declaration needs to be re-applied
51+
-- using HEX for consistency and slight compression
52+
ADD COLUMN token VARCHAR(63) COLLATE utf8mb4_unicode_ci GENERATED ALWAYS AS
53+
(CONCAT(HEX(checksum), COALESCE(CONCAT('-', HEX(TO_SECONDS(secrets_updated_at)), '-', HEX(MICROSECOND(secrets_updated_at)))), '')) VIRTUAL NULL`)
4254
if err != nil {
4355
return fmt.Errorf("failed to add secrets_updated_at to host_mdm_apple_declarations table: %w", err)
4456
}

‎server/datastore/mysql/schema.sql

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,8 @@ CREATE TABLE `host_mdm_apple_declarations` (
423423
`declaration_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
424424
`declaration_identifier` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL,
425425
`declaration_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
426-
`secrets_updated_at` timestamp(6) NULL DEFAULT NULL,
426+
`secrets_updated_at` datetime(6) DEFAULT NULL,
427+
`token` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci GENERATED ALWAYS AS (concat(hex(`checksum`),coalesce(concat(_utf8mb4'-',hex(to_seconds(`secrets_updated_at`)),_utf8mb4'-',hex(microsecond(`secrets_updated_at`)))),_utf8mb4'')) VIRTUAL,
427428
PRIMARY KEY (`host_uuid`,`declaration_uuid`),
428429
KEY `status` (`status`),
429430
KEY `operation_type` (`operation_type`),
@@ -446,7 +447,7 @@ CREATE TABLE `host_mdm_apple_profiles` (
446447
`profile_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
447448
`created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
448449
`updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
449-
`secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
450+
`secrets_updated_at` datetime(6) DEFAULT NULL,
450451
PRIMARY KEY (`host_uuid`,`profile_uuid`),
451452
KEY `status` (`status`),
452453
KEY `operation_type` (`operation_type`),
@@ -854,7 +855,7 @@ CREATE TABLE `mdm_apple_configuration_profiles` (
854855
`uploaded_at` timestamp(6) NULL DEFAULT NULL,
855856
`checksum` binary(16) NOT NULL,
856857
`profile_uuid` varchar(37) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
857-
`secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
858+
`secrets_updated_at` datetime(6) DEFAULT NULL,
858859
PRIMARY KEY (`profile_uuid`),
859860
UNIQUE KEY `idx_mdm_apple_config_prof_team_identifier` (`team_id`,`identifier`),
860861
UNIQUE KEY `idx_mdm_apple_config_prof_team_name` (`team_id`,`name`),
@@ -884,7 +885,8 @@ CREATE TABLE `mdm_apple_declarations` (
884885
`created_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
885886
`uploaded_at` timestamp(6) NULL DEFAULT NULL,
886887
`auto_increment` bigint NOT NULL AUTO_INCREMENT,
887-
`secrets_updated_at` timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
888+
`secrets_updated_at` datetime(6) DEFAULT NULL,
889+
`token` varchar(63) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci GENERATED ALWAYS AS (concat(hex(`checksum`),coalesce(concat(_utf8mb4'-',hex(to_seconds(`secrets_updated_at`)),_utf8mb4'-',hex(microsecond(`secrets_updated_at`)))),_utf8mb4'')) VIRTUAL,
888890
PRIMARY KEY (`declaration_uuid`),
889891
UNIQUE KEY `idx_mdm_apple_declaration_team_identifier` (`team_id`,`identifier`),
890892
UNIQUE KEY `idx_mdm_apple_declaration_team_name` (`team_id`,`name`),

‎server/fleet/apple_mdm.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ type MDMAppleProfilePayload struct {
321321
HostUUID string `db:"host_uuid"`
322322
HostPlatform string `db:"host_platform"`
323323
Checksum []byte `db:"checksum"`
324-
SecretsUpdatedAt time.Time `db:"secrets_updated_at"`
324+
SecretsUpdatedAt *time.Time `db:"secrets_updated_at"`
325325
Status *MDMDeliveryStatus `db:"status" json:"status"`
326326
OperationType MDMOperationType `db:"operation_type"`
327327
Detail string `db:"detail"`
@@ -587,8 +587,13 @@ type MDMAppleDeclaration struct {
587587
// RawJSON is the raw JSON content of the declaration
588588
RawJSON json.RawMessage `db:"raw_json" json:"-"`
589589

590+
// Token is used to identify if declaration needs to be re-applied.
591+
// It contains the checksum of the JSON contents and seconds-microseconds of secrets updated timestamp (if secret variables are present).
592+
Token string `db:"token" json:"-"`
593+
590594
// Checksum is a checksum of the JSON contents
591-
Checksum string `db:"checksum" json:"-"`
595+
// BOZO: Needed?
596+
// Checksum string `db:"checksum" json:"-"`
592597

593598
// labels associated with this Declaration
594599
LabelsIncludeAll []ConfigurationProfileLabel `db:"-" json:"labels_include_all,omitempty"`
@@ -762,7 +767,7 @@ type MDMAppleDDMManifest struct {
762767
// https://developer.apple.com/documentation/devicemanagement/declarationitemsresponse
763768
type MDMAppleDDMDeclarationItem struct {
764769
Identifier string `db:"identifier"`
765-
ServerToken string `db:"checksum"`
770+
ServerToken string `db:"token"`
766771
}
767772

768773
// MDMAppleDDMDeclarationResponse represents a declaration in the datastore. It is used for the DDM

‎server/fleet/apple_mdm_test.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,70 @@ func TestMDMProfileIsWithinGracePeriod(t *testing.T) {
419419
}
420420
}
421421

422+
func TestMDMAppleHostDeclarationEqual(t *testing.T) {
423+
t.Parallel()
424+
425+
// This test is intended to ensure that the Equal method on MDMAppleHostDeclaration is updated when new fields are added.
426+
// The Equal method is used to identify whether database update is needed.
427+
428+
items := [...]MDMAppleHostDeclaration{{}, {}}
429+
430+
numberOfFields := 0
431+
for i := 0; i < len(items); i++ {
432+
rValue := reflect.ValueOf(&items[i]).Elem()
433+
numberOfFields = rValue.NumField()
434+
for j := 0; j < numberOfFields; j++ {
435+
field := rValue.Field(j)
436+
switch field.Kind() {
437+
case reflect.String:
438+
valueToSet := fmt.Sprintf("test %d", i)
439+
field.SetString(valueToSet)
440+
case reflect.Int:
441+
field.SetInt(int64(i))
442+
case reflect.Bool:
443+
field.SetBool(i%2 == 0)
444+
case reflect.Pointer:
445+
field.Set(reflect.New(field.Type().Elem()))
446+
default:
447+
t.Fatalf("unhandled field type %s", field.Kind())
448+
}
449+
}
450+
}
451+
452+
status0 := MDMDeliveryStatus("status")
453+
status1 := MDMDeliveryStatus("status")
454+
items[0].Status = &status0
455+
assert.False(t, items[0].Equal(items[1]))
456+
457+
// Set known fields to be equal
458+
fieldsInEqualMethod := 0
459+
items[1].HostUUID = items[0].HostUUID
460+
fieldsInEqualMethod++
461+
items[1].DeclarationUUID = items[0].DeclarationUUID
462+
fieldsInEqualMethod++
463+
items[1].Name = items[0].Name
464+
fieldsInEqualMethod++
465+
items[1].Identifier = items[0].Identifier
466+
fieldsInEqualMethod++
467+
items[1].OperationType = items[0].OperationType
468+
fieldsInEqualMethod++
469+
items[1].Detail = items[0].Detail
470+
fieldsInEqualMethod++
471+
items[1].Checksum = items[0].Checksum
472+
fieldsInEqualMethod++
473+
items[1].Status = &status1
474+
fieldsInEqualMethod++
475+
items[1].SecretsUpdatedAt = items[0].SecretsUpdatedAt
476+
fieldsInEqualMethod++
477+
assert.Equal(t, fieldsInEqualMethod, numberOfFields, "MDMAppleHostDeclaration.Equal needs to be updated for new/updated field(s)")
478+
assert.True(t, items[0].Equal(items[1]))
479+
480+
// Set pointers to nil
481+
items[0].Status = nil
482+
items[1].Status = nil
483+
assert.True(t, items[0].Equal(items[1]))
484+
}
485+
422486
func TestConfigurationProfileLabelEqual(t *testing.T) {
423487
t.Parallel()
424488

‎server/fleet/mdm.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ func NewMDMConfigProfilePayloadFromAppleDDM(decl *MDMAppleDeclaration) *MDMConfi
492492
Name: decl.Name,
493493
Identifier: decl.Identifier,
494494
Platform: "darwin",
495-
Checksum: []byte(decl.Checksum),
495+
Checksum: []byte(decl.Token),
496496
CreatedAt: decl.CreatedAt,
497497
UploadedAt: decl.UploadedAt,
498498
LabelsIncludeAll: decl.LabelsIncludeAll,

‎server/service/apple_mdm.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4262,7 +4262,7 @@ func (svc *MDMAppleDDMService) handleActivationDeclaration(ctx context.Context,
42624262
},
42634263
"ServerToken": "%s",
42644264
"Type": "com.apple.activation.simple"
4265-
}`, parts[2], references, d.Checksum)
4265+
}`, parts[2], references, d.Token)
42664266

42674267
return []byte(response), nil
42684268
}
@@ -4285,7 +4285,7 @@ func (svc *MDMAppleDDMService) handleConfigurationDeclaration(ctx context.Contex
42854285
if err := json.Unmarshal([]byte(expanded), &tempd); err != nil {
42864286
return nil, ctxerr.Wrap(ctx, err, "unmarshaling stored declaration")
42874287
}
4288-
tempd["ServerToken"] = d.Checksum
4288+
tempd["ServerToken"] = d.Token
42894289

42904290
b, err := json.Marshal(tempd)
42914291
if err != nil {

‎server/service/integration_mdm_ddm_test.go

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -484,10 +484,6 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() {
484484
require.Equal(t, d.Identifier, m.Identifier)
485485
}
486486
}
487-
calcChecksum := func(source []byte) string {
488-
csum := fmt.Sprintf("%x", md5.Sum(source)) //nolint:gosec
489-
return strings.ToUpper(csum)
490-
}
491487

492488
tmpl := `
493489
{
@@ -516,17 +512,6 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() {
516512
decls[1] = []byte(strings.ReplaceAll(string(decls[1]), myBash, "$"+fleet.ServerSecretPrefix+"BASH"))
517513
secretProfile := decls[2]
518514
decls[2] = []byte("${" + fleet.ServerSecretPrefix + "PROFILE}")
519-
declsByChecksum := map[string]fleet.MDMAppleDeclaration{
520-
calcChecksum(decls[0]): {
521-
Identifier: "com.fleet.config0",
522-
},
523-
calcChecksum(decls[1]): {
524-
Identifier: "com.fleet.config1",
525-
},
526-
calcChecksum(decls[2]): {
527-
Identifier: "com.fleet.config2",
528-
},
529-
}
530515

531516
// Create declarations
532517
// First dry run
@@ -585,7 +570,7 @@ SELECT
585570
identifier,
586571
name,
587572
raw_json,
588-
checksum,
573+
token,
589574
created_at,
590575
uploaded_at
591576
FROM mdm_apple_declarations
@@ -599,19 +584,28 @@ WHERE name = ?`
599584
}
600585
nameToIdentifier := make(map[string]string, 3)
601586
nameToUUID := make(map[string]string, 3)
587+
declsByChecksum := map[string]fleet.MDMAppleDeclaration{}
602588
decl := getDeclaration(t, "N0")
603589
nameToIdentifier["N0"] = decl.Identifier
604590
nameToUUID["N0"] = decl.DeclarationUUID
591+
declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{
592+
Identifier: "com.fleet.config0",
593+
}
605594
decl = getDeclaration(t, "N1")
606595
assert.NotContains(t, string(decl.RawJSON), myBash)
607596
assert.Contains(t, string(decl.RawJSON), "$"+fleet.ServerSecretPrefix+"BASH")
608597
nameToIdentifier["N1"] = decl.Identifier
609598
nameToUUID["N1"] = decl.DeclarationUUID
599+
declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{
600+
Identifier: "com.fleet.config1",
601+
}
610602
decl = getDeclaration(t, "N2")
611603
assert.Equal(t, string(decl.RawJSON), "${"+fleet.ServerSecretPrefix+"PROFILE}")
612604
nameToIdentifier["N2"] = decl.Identifier
613605
nameToUUID["N2"] = decl.DeclarationUUID
614-
606+
declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{
607+
Identifier: "com.fleet.config2",
608+
}
615609
// trigger a profile sync
616610
s.awaitTriggerProfileSchedule(t)
617611

0 commit comments

Comments
 (0)