diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index f5ca37855d8a..4061d3ed477c 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -214,7 +214,8 @@ SELECT mobileconfig, checksum, created_at, - uploaded_at + uploaded_at, + secrets_updated_at FROM mdm_apple_configuration_profiles WHERE @@ -280,7 +281,8 @@ SELECT checksum, token, created_at, - uploaded_at + uploaded_at, + secrets_updated_at FROM mdm_apple_declarations WHERE @@ -4221,15 +4223,17 @@ INSERT INTO mdm_apple_declarations ( name, raw_json, checksum, + secrets_updated_at, uploaded_at, team_id ) VALUES ( - ?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP(),? + ?,?,?,?,UNHEX(?),?,CURRENT_TIMESTAMP(),? ) ON DUPLICATE KEY UPDATE uploaded_at = IF(checksum = VALUES(checksum) AND name = VALUES(name), uploaded_at, CURRENT_TIMESTAMP()), checksum = VALUES(checksum), + secrets_updated_at = VALUES(secrets_updated_at), name = VALUES(name), identifier = VALUES(identifier), raw_json = VALUES(raw_json) @@ -4337,6 +4341,7 @@ WHERE d.Name, d.RawJSON, checksum, + d.SecretsUpdatedAt, declTeamID); err != nil || strings.HasPrefix(ds.testBatchSetMDMAppleProfilesErr, "insert") { if err == nil { err = errors.New(ds.testBatchSetMDMAppleProfilesErr) @@ -4412,8 +4417,9 @@ INSERT INTO mdm_apple_declarations ( name, raw_json, checksum, + secrets_updated_at, uploaded_at) -(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE +(SELECT ?,?,?,?,?,UNHEX(?),?,CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? ) AND NOT EXISTS ( @@ -4433,8 +4439,9 @@ INSERT INTO mdm_apple_declarations ( name, raw_json, checksum, + secrets_updated_at, uploaded_at) -(SELECT ?,?,?,?,?,UNHEX(?),CURRENT_TIMESTAMP() FROM DUAL WHERE +(SELECT ?,?,?,?,?,UNHEX(?),?,CURRENT_TIMESTAMP() FROM DUAL WHERE NOT EXISTS ( SELECT 1 FROM mdm_windows_configuration_profiles WHERE name = ? AND team_id = ? ) AND NOT EXISTS ( @@ -4463,7 +4470,8 @@ func (ds *Datastore) insertOrUpsertMDMAppleDeclaration(ctx context.Context, insO err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { res, err := tx.ExecContext(ctx, insOrUpsertStmt, - declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, checksum, declaration.Name, tmID, declaration.Name, tmID) + declUUID, tmID, declaration.Identifier, declaration.Name, declaration.RawJSON, checksum, declaration.SecretsUpdatedAt, + declaration.Name, tmID, declaration.Name, tmID) if err != nil { switch { case IsDuplicate(err): @@ -4643,7 +4651,7 @@ func batchSetDeclarationLabelAssociationsDB(ctx context.Context, tx sqlx.ExtCont func (ds *Datastore) MDMAppleDDMDeclarationsToken(ctx context.Context, hostUUID string) (*fleet.MDMAppleDDMDeclarationsToken, error) { const stmt = ` SELECT - COALESCE(MD5((count(0) + GROUP_CONCAT(HEX(hmad.token) + COALESCE(MD5((count(0) + GROUP_CONCAT(HEX(mad.token) ORDER BY mad.uploaded_at DESC separator ''))), '') AS token, COALESCE(MAX(mad.created_at), NOW()) AS latest_created_timestamp @@ -4672,7 +4680,7 @@ WHERE func (ds *Datastore) MDMAppleDDMDeclarationItems(ctx context.Context, hostUUID string) ([]fleet.MDMAppleDDMDeclarationItem, error) { const stmt = ` SELECT - HEX(token) as token, + HEX(mad.token) as token, mad.identifier FROM host_mdm_apple_declarations hmad diff --git a/server/datastore/mysql/secret_variables.go b/server/datastore/mysql/secret_variables.go index 7a0bbf0fc5b7..14aa17cab7af 100644 --- a/server/datastore/mysql/secret_variables.go +++ b/server/datastore/mysql/secret_variables.go @@ -157,7 +157,7 @@ func (ds *Datastore) expandEmbeddedSecrets(ctx context.Context, document string) return val, ok }) - return expanded, nil, nil + return expanded, secrets, nil } func (ds *Datastore) ExpandEmbeddedSecretsAndUpdatedAt(ctx context.Context, document string) (string, *time.Time, error) { diff --git a/server/datastore/mysql/secret_variables_test.go b/server/datastore/mysql/secret_variables_test.go index b52932b6bf2b..a4ad33e1db69 100644 --- a/server/datastore/mysql/secret_variables_test.go +++ b/server/datastore/mysql/secret_variables_test.go @@ -137,7 +137,7 @@ Hello doc${FLEET_SECRET_INVALID}. $FLEET_SECRET_ALSO_INVALID func testExpandEmbeddedSecrets(t *testing.T, ds *Datastore) { noSecrets := ` -This document contains to fleet secrets. +This document contains no fleet secrets. $FLEET_VAR_XX $HOSTNAME ${SOMETHING_ELSE} ` @@ -172,10 +172,18 @@ Hello doc${FLEET_SECRET_INVALID}. $FLEET_SECRET_ALSO_INVALID expanded, err := ds.ExpandEmbeddedSecrets(ctx, noSecrets) require.NoError(t, err) require.Equal(t, noSecrets, expanded) + expanded, secretsUpdatedAt, err := ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, noSecrets) + require.NoError(t, err) + require.Equal(t, noSecrets, expanded) + assert.Nil(t, secretsUpdatedAt) expanded, err = ds.ExpandEmbeddedSecrets(ctx, validSecret) require.NoError(t, err) require.Equal(t, validSecretExpanded, expanded) + expanded, secretsUpdatedAt, err = ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, validSecret) + require.NoError(t, err) + require.Equal(t, validSecretExpanded, expanded) + assert.NotNil(t, secretsUpdatedAt) _, err = ds.ExpandEmbeddedSecrets(ctx, invalidSecret) require.ErrorContains(t, err, "$FLEET_SECRET_INVALID") diff --git a/server/fleet/apple_mdm.go b/server/fleet/apple_mdm.go index 502cb899618b..99b7fff88d14 100644 --- a/server/fleet/apple_mdm.go +++ b/server/fleet/apple_mdm.go @@ -600,8 +600,9 @@ type MDMAppleDeclaration struct { LabelsIncludeAny []ConfigurationProfileLabel `db:"-" json:"labels_include_any,omitempty"` LabelsExcludeAny []ConfigurationProfileLabel `db:"-" json:"labels_exclude_any,omitempty"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UploadedAt time.Time `db:"uploaded_at" json:"uploaded_at"` + SecretsUpdatedAt *time.Time `db:"secrets_updated_at" json:"-"` } type MDMAppleRawDeclaration struct { diff --git a/server/fleet/mdm.go b/server/fleet/mdm.go index b441067398b8..0690d5ed96bd 100644 --- a/server/fleet/mdm.go +++ b/server/fleet/mdm.go @@ -437,10 +437,11 @@ type MDMProfileBatchPayload struct { // Deprecated: Labels is the backwards-compatible way of specifying // LabelsIncludeAll. - Labels []string `json:"labels,omitempty"` - LabelsIncludeAll []string `json:"labels_include_all,omitempty"` - LabelsIncludeAny []string `json:"labels_include_any,omitempty"` - LabelsExcludeAny []string `json:"labels_exclude_any,omitempty"` + Labels []string `json:"labels,omitempty"` + LabelsIncludeAll []string `json:"labels_include_all,omitempty"` + LabelsIncludeAny []string `json:"labels_include_any,omitempty"` + LabelsExcludeAny []string `json:"labels_exclude_any,omitempty"` + SecretsUpdatedAt *time.Time `json:"-"` } func NewMDMConfigProfilePayloadFromWindows(cp *MDMWindowsConfigProfile) *MDMConfigProfilePayload { diff --git a/server/service/apple_mdm.go b/server/service/apple_mdm.go index 14dcb32215aa..fbca4426bcf9 100644 --- a/server/service/apple_mdm.go +++ b/server/service/apple_mdm.go @@ -513,7 +513,7 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i return nil, err } - dataWithSecrets, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(data)) + dataWithSecrets, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(data)) if err != nil { return nil, fleet.NewInvalidArgumentError("profile", err.Error()) } @@ -534,6 +534,7 @@ func (svc *Service) NewMDMAppleDeclaration(ctx context.Context, teamID uint, r i } d := fleet.NewMDMAppleDeclaration(data, tmID, name, rawDecl.Type, rawDecl.Identifier) + d.SecretsUpdatedAt = secretsUpdatedAt switch labelsMembershipMode { case fleet.LabelsIncludeAny: diff --git a/server/service/integration_mdm_ddm_test.go b/server/service/integration_mdm_ddm_test.go index 5dc805d3ac04..9f58fc3b24e8 100644 --- a/server/service/integration_mdm_ddm_test.go +++ b/server/service/integration_mdm_ddm_test.go @@ -472,15 +472,20 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariables() { _, mdmDevice := createHostThenEnrollMDM(s.ds, s.server.URL, t) checkDeclarationItemsResp := func(t *testing.T, r fleet.MDMAppleDDMDeclarationItemsResponse, expectedDeclTok string, - expectedDeclsByChecksum map[string]fleet.MDMAppleDeclaration) { + expectedDeclsByToken map[string]fleet.MDMAppleDeclaration) { require.Equal(t, expectedDeclTok, r.DeclarationsToken) require.NotEmpty(t, r.Declarations.Activations) require.Empty(t, r.Declarations.Assets) require.Empty(t, r.Declarations.Management) - require.Len(t, r.Declarations.Configurations, len(expectedDeclsByChecksum)) + require.Len(t, r.Declarations.Configurations, len(expectedDeclsByToken)) for _, m := range r.Declarations.Configurations { - d, ok := expectedDeclsByChecksum[m.ServerToken] - require.True(t, ok) + d, ok := expectedDeclsByToken[m.ServerToken] + if !ok { + for k := range expectedDeclsByToken { + t.Logf("expected token: %x", k) + } + } + require.True(t, ok, "server token %x not found for %s", m.ServerToken, m.Identifier) require.Equal(t, d.Identifier, m.Identifier) } } @@ -570,9 +575,11 @@ SELECT identifier, name, raw_json, - token, + HEX(checksum) as checksum, + HEX(token) as token, created_at, - uploaded_at + uploaded_at, + secrets_updated_at FROM mdm_apple_declarations WHERE name = ?` @@ -584,11 +591,11 @@ WHERE name = ?` } nameToIdentifier := make(map[string]string, 3) nameToUUID := make(map[string]string, 3) - declsByChecksum := map[string]fleet.MDMAppleDeclaration{} + declsByToken := map[string]fleet.MDMAppleDeclaration{} decl := getDeclaration(t, "N0") nameToIdentifier["N0"] = decl.Identifier nameToUUID["N0"] = decl.DeclarationUUID - declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{ + declsByToken[decl.Token] = fleet.MDMAppleDeclaration{ Identifier: "com.fleet.config0", } decl = getDeclaration(t, "N1") @@ -596,14 +603,14 @@ WHERE name = ?` assert.Contains(t, string(decl.RawJSON), "$"+fleet.ServerSecretPrefix+"BASH") nameToIdentifier["N1"] = decl.Identifier nameToUUID["N1"] = decl.DeclarationUUID - declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{ + declsByToken[decl.Token] = fleet.MDMAppleDeclaration{ Identifier: "com.fleet.config1", } decl = getDeclaration(t, "N2") assert.Equal(t, string(decl.RawJSON), "${"+fleet.ServerSecretPrefix+"PROFILE}") nameToIdentifier["N2"] = decl.Identifier nameToUUID["N2"] = decl.DeclarationUUID - declsByChecksum[decl.Token] = fleet.MDMAppleDeclaration{ + declsByToken[decl.Token] = fleet.MDMAppleDeclaration{ Identifier: "com.fleet.config2", } // trigger a profile sync @@ -618,7 +625,7 @@ WHERE name = ?` r, err = mdmDevice.DeclarativeManagement("declaration-items") require.NoError(t, err) itemsResp := parseDeclarationItemsResp(t, r) - checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByChecksum) + checkDeclarationItemsResp(t, itemsResp, currDeclToken, declsByToken) // Now, retrieve the declaration configuration profiles declarationPath := fmt.Sprintf("declaration/configuration/%s", nameToIdentifier["N0"]) diff --git a/server/service/integration_mdm_profiles_test.go b/server/service/integration_mdm_profiles_test.go index 9f626d930070..976be8a815db 100644 --- a/server/service/integration_mdm_profiles_test.go +++ b/server/service/integration_mdm_profiles_test.go @@ -259,12 +259,47 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() { s.checkMDMProfilesSummaries(t, nil, expectedNoTeamSummary, &expectedNoTeamSummary) // empty because host was transferred s.checkMDMProfilesSummaries(t, &tm.ID, expectedTeamSummary, &expectedTeamSummary) // host still verifying team profiles - // with no changes + // Upload the same profiles again. No changes expected. + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, + "team_id", fmt.Sprint(tm.ID)) s.awaitTriggerProfileSchedule(t) installs, removes = checkNextPayloads(t, mdmDevice, false) require.Empty(t, installs) require.Empty(t, removes) + // Change the secret variable and upload the profiles again. We should see the profile with updated secret installed. + secretName = "newSecretName" + req = secretVariablesRequest{ + SecretVariables: []fleet.SecretVariable{ + { + Name: "FLEET_SECRET_NAME", + Value: secretName, // changed + }, + { + Name: "FLEET_SECRET_PROFILE", + Value: secretProfile, // did not change + }, + }, + } + s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp) + s.Do("POST", "/api/v1/fleet/mdm/apple/profiles/batch", batchSetMDMAppleProfilesRequest{Profiles: teamProfiles}, http.StatusNoContent, + "team_id", fmt.Sprint(tm.ID)) + s.awaitTriggerProfileSchedule(t) + installs, removes = checkNextPayloads(t, mdmDevice, false) + // Manually replace the expected secret variables in the profile + wantTeamProfilesChanged := [][]byte{ + teamProfiles[1], + } + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_IDENTIFIER", + secretIdentifier)) + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "${FLEET_SECRET_TYPE}", secretType)) + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_NAME", secretName)) + // verify that we should install the team profiles + s.signedProfilesMatch(wantTeamProfilesChanged, installs) + wantTeamProfiles[1] = wantTeamProfilesChanged[0] + // No profiles should be deleted + assert.Empty(t, removes) + // Clear the profiles using the new (non-deprecated) endpoint. s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchSetMDMProfilesRequest{Profiles: nil}, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID), "dry_run", "true") @@ -295,6 +330,47 @@ func (s *integrationMDMTestSuite) TestAppleProfileManagement() { // verify that we should install the team profiles s.signedProfilesMatch(wantTeamProfiles, installs) + // Upload the same profiles again. No changes expected. + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID), "dry_run", "true") + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID)) + s.awaitTriggerProfileSchedule(t) + installs, removes = checkNextPayloads(t, mdmDevice, false) + require.Empty(t, installs) + require.Empty(t, removes) + + // Change the secret variable and upload the profiles again. We should see the profile with updated secret installed. + secretName = "new2SecretName" + req = secretVariablesRequest{ + SecretVariables: []fleet.SecretVariable{ + { + Name: "FLEET_SECRET_NAME", + Value: secretName, // changed + }, + { + Name: "FLEET_SECRET_PROFILE", + Value: secretProfile, // did not change + }, + }, + } + s.DoJSON("PUT", "/api/latest/fleet/spec/secret_variables", req, http.StatusOK, &secretResp) + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID), "dry_run", "true") + s.Do("POST", "/api/v1/fleet/mdm/profiles/batch", batchRequest, http.StatusNoContent, "team_id", fmt.Sprint(tm.ID)) + s.awaitTriggerProfileSchedule(t) + installs, removes = checkNextPayloads(t, mdmDevice, false) + // Manually replace the expected secret variables in the profile + wantTeamProfilesChanged = [][]byte{ + teamProfiles[1], + } + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_IDENTIFIER", + secretIdentifier)) + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "${FLEET_SECRET_TYPE}", secretType)) + wantTeamProfilesChanged[0] = []byte(strings.ReplaceAll(string(wantTeamProfilesChanged[0]), "$FLEET_SECRET_NAME", secretName)) + // verify that we should install the team profiles + s.signedProfilesMatch(wantTeamProfilesChanged, installs) + wantTeamProfiles[1] = wantTeamProfilesChanged[0] + // No profiles should be deleted + assert.Empty(t, removes) + var hostResp getHostResponse s.DoJSON("GET", fmt.Sprintf("/api/v1/fleet/hosts/%d", host.ID), getHostRequest{}, http.StatusOK, &hostResp) require.NotEmpty(t, hostResp.Host.MDM.Profiles) @@ -5291,6 +5367,9 @@ func (s *integrationMDMTestSuite) TestAppleDDMSecretVariablesUpload() { getProfileContents := func(profileUUID string) string { profile, err := s.ds.GetMDMAppleDeclaration(context.Background(), profileUUID) require.NoError(s.T(), err) + // Since our DDM profiles contain secrets, the checksum and token should be different + assert.NotNil(s.T(), profile.SecretsUpdatedAt) + assert.NotEqual(s.T(), profile.Checksum, profile.Token) return string(profile.RawJSON) } @@ -5424,6 +5503,7 @@ func (s *integrationMDMTestSuite) TestAppleConfigSecretVariablesUpload() { getProfileContents := func(profileUUID string) string { profile, err := s.ds.GetMDMAppleConfigProfile(context.Background(), profileUUID) require.NoError(s.T(), err) + assert.NotNil(s.T(), profile.SecretsUpdatedAt) return string(profile.Mobileconfig) } diff --git a/server/service/mdm.go b/server/service/mdm.go index 8fdc0da52b3a..406c79441777 100644 --- a/server/service/mdm.go +++ b/server/service/mdm.go @@ -1636,10 +1636,11 @@ func (svc *Service) BatchSetMDMProfiles( // In order to map the expanded profiles back to the original profiles, we will use the index. profilesWithSecrets := make(map[int]fleet.MDMProfileBatchPayload, len(profiles)) for i, p := range profiles { - expanded, err := svc.ds.ExpandEmbeddedSecrets(ctx, string(p.Contents)) + expanded, secretsUpdatedAt, err := svc.ds.ExpandEmbeddedSecretsAndUpdatedAt(ctx, string(p.Contents)) if err != nil { return err } + p.SecretsUpdatedAt = secretsUpdatedAt pCopy := p // If the profile does not contain secrets, then expanded and original content point to the same slice/memory location. pCopy.Contents = []byte(expanded) @@ -1912,6 +1913,7 @@ func getAppleProfiles( } mdmDecl := fleet.NewMDMAppleDeclaration(prof.Contents, tmID, prof.Name, rawDecl.Type, rawDecl.Identifier) + mdmDecl.SecretsUpdatedAt = prof.SecretsUpdatedAt for _, labelName := range prof.LabelsIncludeAll { if lbl, ok := labelMap[labelName]; ok { declLabel := fleet.ConfigurationProfileLabel{ @@ -1967,6 +1969,7 @@ func getAppleProfiles( } mdmProf, err := fleet.NewMDMAppleConfigProfile(prof.Contents, tmID) + mdmProf.SecretsUpdatedAt = prof.SecretsUpdatedAt if err != nil { return nil, nil, ctxerr.Wrap(ctx, fleet.NewInvalidArgumentError(prof.Name, err.Error()),