diff --git a/changes/24385-automatic-install-custom-packages b/changes/24385-automatic-install-custom-packages new file mode 100644 index 000000000000..b36526d1687f --- /dev/null +++ b/changes/24385-automatic-install-custom-packages @@ -0,0 +1 @@ +* Added capability to automatically generate "trigger policies" for custom software packages. diff --git a/docker-compose.yml b/docker-compose.yml index 6a393010e035..62dc87a9db7c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ --- -version: "2" services: # To test with MariaDB, set FLEET_MYSQL_IMAGE to mariadb:10.6 or the like (note MariaDB is not # officially supported). diff --git a/ee/server/service/software_installers.go b/ee/server/service/software_installers.go index 67c81f7d44ba..67fdd7f340c3 100644 --- a/ee/server/service/software_installers.go +++ b/ee/server/service/software_installers.go @@ -37,6 +37,14 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. return err } + if payload.AutomaticInstall { + // Currently, same write permissions are applied on software and policies, + // but leaving this here in case it changes in the future. + if err := svc.authz.Authorize(ctx, &fleet.Policy{PolicyData: fleet.PolicyData{TeamID: payload.TeamID}}, fleet.ActionWrite); err != nil { + return err + } + } + // validate labels before we do anything else validatedLabels, err := ValidateSoftwareLabels(ctx, svc, payload.LabelsIncludeAny, payload.LabelsExcludeAny) if err != nil { @@ -61,13 +69,29 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. return ctxerr.Wrap(ctx, err, "adding metadata to payload") } + if payload.AutomaticInstall { + switch { + // + // For "msi", addMetadataToSoftwarePayload fails before this point if product code cannot be extracted. + // + case payload.Extension == "exe": + return &fleet.BadRequestError{ + Message: "Couldn't add. Fleet can't create a policy to detect existing installations for .exe packages. Please add the software, add a custom policy, and enable the install software policy automation.", + } + case payload.Extension == "pkg" && payload.BundleIdentifier == "": + // For pkgs without bundle identifier the request usually fails before reaching this point, + // but addMetadataToSoftwarePayload may not fail if the package has "package IDs" but not a "bundle identifier", + // in which case we want to fail here because we cannot generate a policy without a bundle identifier. + return &fleet.BadRequestError{ + Message: "Couldn't add. Policy couldn't be created because bundle identifier can't be extracted.", + } + } + } + if err := svc.storeSoftware(ctx, payload); err != nil { return ctxerr.Wrap(ctx, err, "storing software installer") } - // TODO: basic validation of install and post-install script (e.g., supported interpreters)? - // TODO: any validation of pre-install query? - // Update $PACKAGE_ID in uninstall script preProcessUninstallScript(payload) @@ -81,8 +105,6 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. } level.Debug(svc.logger).Log("msg", "software installer uploaded", "installer_id", installerID) - // TODO: QA what breaks when you have a software title with no versions? - var teamName *string if payload.TeamID != nil && *payload.TeamID != 0 { t, err := svc.ds.Team(ctx, *payload.TeamID) @@ -92,7 +114,6 @@ func (svc *Service) UploadSoftwareInstaller(ctx context.Context, payload *fleet. teamName = &t.Name } - // Create activity actLabelsIncl, actLabelsExcl := activitySoftwareLabelsFromValidatedLabels(payload.ValidatedLabels) if err := svc.NewActivity(ctx, vc.User, fleet.ActivityTypeAddedSoftware{ SoftwareTitle: payload.Title, @@ -1241,7 +1262,7 @@ func (svc *Service) addMetadataToSoftwarePayload(ctx context.Context, payload *f if len(meta.PackageIDs) == 0 { return "", &fleet.BadRequestError{ - Message: fmt.Sprintf("Couldn't add. Fleet couldn't read the package IDs, product code, or name from %s.", payload.Filename), + Message: "Couldn't add. Unable to extract necessary metadata.", InternalErr: ctxerr.New(ctx, "extracting package IDs from installer metadata"), } } diff --git a/pkg/automatic_policy/automatic_policy.go b/pkg/automatic_policy/automatic_policy.go new file mode 100644 index 000000000000..108928e14a6a --- /dev/null +++ b/pkg/automatic_policy/automatic_policy.go @@ -0,0 +1,116 @@ +// Package automatic_policy generates "trigger policies" from metadata of software packages. +package automatic_policy + +import ( + "errors" + "fmt" +) + +// PolicyData contains generated data for a policy to trigger installation of a software package. +type PolicyData struct { + // Name is the generated name of the policy. + Name string + // Query is the generated SQL/sqlite of the policy. + Query string + // Description is the generated description for the policy. + Description string + // Platform is the target platform for the policy. + Platform string +} + +// InstallerMetadata contains the metadata of a software package used to generate the policies. +type InstallerMetadata struct { + // Title is the software title extracted from a software package. + Title string + // Extension is the extension of the software package. + Extension string + // BundleIdentifier contains the bundle identifier for 'pkg' packages. + BundleIdentifier string + // PackageIDs contains the product code for 'msi' packages. + PackageIDs []string +} + +var ( + // ErrExtensionNotSupported is returned if the extension is not supported to generate automatic policies. + ErrExtensionNotSupported = errors.New("extension not supported") + // ErrMissingBundleIdentifier is returned if the software extension is "pkg" and a bundle identifier was not extracted from the installer. + ErrMissingBundleIdentifier = errors.New("missing bundle identifier") + // ErrMissingProductCode is returned if the software extension is "msi" and a product code was not extracted from the installer. + ErrMissingProductCode = errors.New("missing product code") + // ErrMissingTitle is returned if a title was not extracted from the installer. + ErrMissingTitle = errors.New("missing title") +) + +// Generate generates the "trigger policy" from the metadata of a software package. +func Generate(metadata InstallerMetadata) (*PolicyData, error) { + switch { + case metadata.Title == "": + return nil, ErrMissingTitle + case metadata.Extension != "pkg" && metadata.Extension != "msi" && metadata.Extension != "deb" && metadata.Extension != "rpm": + return nil, ErrExtensionNotSupported + case metadata.Extension == "pkg" && metadata.BundleIdentifier == "": + return nil, ErrMissingBundleIdentifier + case metadata.Extension == "msi" && (len(metadata.PackageIDs) == 0 || metadata.PackageIDs[0] == ""): + return nil, ErrMissingProductCode + } + + name := fmt.Sprintf("[Install software] %s (%s)", metadata.Title, metadata.Extension) + + description := fmt.Sprintf("Policy triggers automatic install of %s on each host that's missing this software.", metadata.Title) + if metadata.Extension == "deb" || metadata.Extension == "rpm" { + basedPrefix := "RPM" + if metadata.Extension == "rpm" { + basedPrefix = "Debian" + } + description += fmt.Sprintf( + "\nSoftware won't be installed on Linux hosts with %s-based distributions because this policy's query is written to always pass on these hosts.", + basedPrefix, + ) + } + + switch metadata.Extension { + case "pkg": + return &PolicyData{ + Name: name, + Query: fmt.Sprintf("SELECT 1 FROM apps WHERE bundle_identifier = '%s';", metadata.BundleIdentifier), + Platform: "darwin", + Description: description, + }, nil + case "msi": + return &PolicyData{ + Name: name, + Query: fmt.Sprintf("SELECT 1 FROM programs WHERE identifying_number = '%s';", metadata.PackageIDs[0]), + Platform: "windows", + Description: description, + }, nil + case "deb": + return &PolicyData{ + Name: name, + Query: fmt.Sprintf( + // First inner SELECT will mark the policies as successful on non-DEB-based hosts. + `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM deb_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM deb_packages WHERE name = '%s' +);`, metadata.Title, + ), + Platform: "linux", + Description: description, + }, nil + case "rpm": + return &PolicyData{ + Name: name, + Query: fmt.Sprintf( + // First inner SELECT will mark the policies as successful on non-RPM-based hosts. + `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM rpm_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM rpm_packages WHERE name = '%s' +);`, metadata.Title), + Platform: "linux", + Description: description, + }, nil + default: + return nil, ErrExtensionNotSupported + } +} diff --git a/pkg/automatic_policy/automatic_policy_test.go b/pkg/automatic_policy/automatic_policy_test.go new file mode 100644 index 000000000000..d00b6a382c6c --- /dev/null +++ b/pkg/automatic_policy/automatic_policy_test.go @@ -0,0 +1,108 @@ +package automatic_policy + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGenerateErrors(t *testing.T) { + _, err := Generate(InstallerMetadata{ + Title: "Foobar", + Extension: "exe", + BundleIdentifier: "", + PackageIDs: []string{"Foobar"}, + }) + require.ErrorIs(t, err, ErrExtensionNotSupported) + + _, err = Generate(InstallerMetadata{ + Title: "Foobar", + Extension: "msi", + BundleIdentifier: "", + PackageIDs: []string{""}, + }) + require.ErrorIs(t, err, ErrMissingProductCode) + _, err = Generate(InstallerMetadata{ + Title: "Foobar", + Extension: "msi", + BundleIdentifier: "", + PackageIDs: []string{}, + }) + require.ErrorIs(t, err, ErrMissingProductCode) + + _, err = Generate(InstallerMetadata{ + Title: "Foobar", + Extension: "pkg", + BundleIdentifier: "", + PackageIDs: []string{""}, + }) + require.ErrorIs(t, err, ErrMissingBundleIdentifier) + + _, err = Generate(InstallerMetadata{ + Title: "", + Extension: "deb", + BundleIdentifier: "", + PackageIDs: []string{""}, + }) + require.ErrorIs(t, err, ErrMissingTitle) +} + +func TestGenerate(t *testing.T) { + policyData, err := Generate(InstallerMetadata{ + Title: "Foobar", + Extension: "pkg", + BundleIdentifier: "com.foo.bar", + PackageIDs: []string{"com.foo.bar"}, + }) + require.NoError(t, err) + require.Equal(t, "[Install software] Foobar (pkg)", policyData.Name) + require.Equal(t, "Policy triggers automatic install of Foobar on each host that's missing this software.", policyData.Description) + require.Equal(t, "darwin", policyData.Platform) + require.Equal(t, "SELECT 1 FROM apps WHERE bundle_identifier = 'com.foo.bar';", policyData.Query) + + policyData, err = Generate(InstallerMetadata{ + Title: "Barfoo", + Extension: "msi", + BundleIdentifier: "", + PackageIDs: []string{"foo"}, + }) + require.NoError(t, err) + require.Equal(t, "[Install software] Barfoo (msi)", policyData.Name) + require.Equal(t, "Policy triggers automatic install of Barfoo on each host that's missing this software.", policyData.Description) + require.Equal(t, "windows", policyData.Platform) + require.Equal(t, "SELECT 1 FROM programs WHERE identifying_number = 'foo';", policyData.Query) + + policyData, err = Generate(InstallerMetadata{ + Title: "Zoobar", + Extension: "deb", + BundleIdentifier: "", + PackageIDs: []string{"Zoobar"}, + }) + require.NoError(t, err) + require.Equal(t, "[Install software] Zoobar (deb)", policyData.Name) + require.Equal(t, `Policy triggers automatic install of Zoobar on each host that's missing this software. +Software won't be installed on Linux hosts with RPM-based distributions because this policy's query is written to always pass on these hosts.`, policyData.Description) + require.Equal(t, "linux", policyData.Platform) + require.Equal(t, `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM deb_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM deb_packages WHERE name = 'Zoobar' +);`, policyData.Query) + + policyData, err = Generate(InstallerMetadata{ + Title: "Barzoo", + Extension: "rpm", + BundleIdentifier: "", + PackageIDs: []string{"Barzoo"}, + }) + require.NoError(t, err) + require.Equal(t, "[Install software] Barzoo (rpm)", policyData.Name) + require.Equal(t, `Policy triggers automatic install of Barzoo on each host that's missing this software. +Software won't be installed on Linux hosts with Debian-based distributions because this policy's query is written to always pass on these hosts.`, policyData.Description) + require.Equal(t, "linux", policyData.Platform) + require.Equal(t, `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM rpm_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM rpm_packages WHERE name = 'Barzoo' +);`, policyData.Query) +} diff --git a/server/datastore/mysql/policies.go b/server/datastore/mysql/policies.go index 967e3ea1743d..61b2187e6af9 100644 --- a/server/datastore/mysql/policies.go +++ b/server/datastore/mysql/policies.go @@ -149,7 +149,7 @@ func (ds *Datastore) SavePolicy(ctx context.Context, p *fleet.Policy, shouldRemo } if p.TeamID != nil { - if err := ds.assertTeamMatches(ctx, *p.TeamID, p.SoftwareInstallerID, p.ScriptID); err != nil { + if err := assertTeamMatches(ctx, ds.writer(ctx), *p.TeamID, p.SoftwareInstallerID, p.ScriptID); err != nil { return ctxerr.Wrap(ctx, err, "save policy") } } @@ -185,10 +185,10 @@ var ( errMismatchedScriptTeam = &fleet.BadRequestError{Message: "script is associated with a different team"} ) -func (ds *Datastore) assertTeamMatches(ctx context.Context, teamID uint, softwareInstallerID *uint, scriptID *uint) error { +func assertTeamMatches(ctx context.Context, db sqlx.QueryerContext, teamID uint, softwareInstallerID *uint, scriptID *uint) error { if softwareInstallerID != nil { var softwareInstallerTeamID uint - err := sqlx.GetContext(ctx, ds.reader(ctx), &softwareInstallerTeamID, "SELECT global_or_team_id FROM software_installers WHERE id = ?", softwareInstallerID) + err := sqlx.GetContext(ctx, db, &softwareInstallerTeamID, "SELECT global_or_team_id FROM software_installers WHERE id = ?", softwareInstallerID) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -202,7 +202,7 @@ func (ds *Datastore) assertTeamMatches(ctx context.Context, teamID uint, softwar if scriptID != nil { var scriptTeamID uint - err := sqlx.GetContext(ctx, ds.reader(ctx), &scriptTeamID, "SELECT global_or_team_id FROM scripts WHERE id = ?", scriptID) + err := sqlx.GetContext(ctx, db, &scriptTeamID, "SELECT global_or_team_id FROM scripts WHERE id = ?", scriptID) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -647,8 +647,12 @@ func (ds *Datastore) PolicyQueriesForHost(ctx context.Context, host *fleet.Host) } func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { + return newTeamPolicy(ctx, ds.writer(ctx), teamID, authorID, args) +} + +func newTeamPolicy(ctx context.Context, db sqlx.ExtContext, teamID uint, authorID *uint, args fleet.PolicyPayload) (*fleet.Policy, error) { if args.QueryID != nil { - q, err := ds.Query(ctx, *args.QueryID) + q, err := query(ctx, db, *args.QueryID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "fetching query from id") } @@ -659,7 +663,7 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u // Check team exists. if teamID > 0 { var ok bool - err := ds.writer(ctx).GetContext(ctx, &ok, `SELECT COUNT(*) = 1 FROM teams WHERE id = ?`, teamID) + err := sqlx.GetContext(ctx, db, &ok, `SELECT COUNT(*) = 1 FROM teams WHERE id = ?`, teamID) if err != nil { return nil, ctxerr.Wrap(ctx, err, "get team id") } @@ -671,11 +675,11 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u // We must normalize the name for full Unicode support (Unicode equivalence). nameUnicode := norm.NFC.String(args.Name) - if err := ds.assertTeamMatches(ctx, teamID, args.SoftwareInstallerID, args.ScriptID); err != nil { + if err := assertTeamMatches(ctx, db, teamID, args.SoftwareInstallerID, args.ScriptID); err != nil { return nil, ctxerr.Wrap(ctx, err, "create team policy") } - res, err := ds.writer(ctx).ExecContext(ctx, + res, err := db.ExecContext(ctx, fmt.Sprintf( `INSERT INTO policies (name, query, description, team_id, resolution, author_id, platforms, critical, calendar_events_enabled, software_installer_id, script_id, checksum) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, %s)`, policiesChecksumComputedColumn(), @@ -695,7 +699,7 @@ func (ds *Datastore) NewTeamPolicy(ctx context.Context, teamID uint, authorID *u if err != nil { return nil, ctxerr.Wrap(ctx, err, "getting last id after inserting policy") } - return policyDB(ctx, ds.writer(ctx), uint(lastIdInt64), &teamID) //nolint:gosec // dismiss G115 + return policyDB(ctx, db, uint(lastIdInt64), &teamID) //nolint:gosec // dismiss G115 } func (ds *Datastore) ListTeamPolicies(ctx context.Context, teamID uint, opts fleet.ListOptions, iopts fleet.ListOptions) (teamPolicies, inheritedPolicies []*fleet.Policy, err error) { diff --git a/server/datastore/mysql/queries.go b/server/datastore/mysql/queries.go index d3a5e4d7dc38..e4a8d4951395 100644 --- a/server/datastore/mysql/queries.go +++ b/server/datastore/mysql/queries.go @@ -423,6 +423,10 @@ func (ds *Datastore) deleteQueryStats(ctx context.Context, queryIDs []uint) { // Query returns a single Query identified by id, if such exists. func (ds *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { + return query(ctx, ds.reader(ctx), id) +} + +func query(ctx context.Context, db sqlx.QueryerContext, id uint) (*fleet.Query, error) { sqlQuery := ` SELECT q.id, @@ -457,14 +461,14 @@ func (ds *Datastore) Query(ctx context.Context, id uint) (*fleet.Query, error) { WHERE q.id = ? ` query := &fleet.Query{} - if err := sqlx.GetContext(ctx, ds.reader(ctx), query, sqlQuery, false, fleet.AggregatedStatsTypeScheduledQuery, id); err != nil { + if err := sqlx.GetContext(ctx, db, query, sqlQuery, false, fleet.AggregatedStatsTypeScheduledQuery, id); err != nil { if err == sql.ErrNoRows { return nil, ctxerr.Wrap(ctx, notFound("Query").WithID(id)) } return nil, ctxerr.Wrap(ctx, err, "selecting query") } - if err := ds.loadPacksForQueries(ctx, []*fleet.Query{query}); err != nil { + if err := loadPacksForQueries(ctx, db, []*fleet.Query{query}); err != nil { return nil, ctxerr.Wrap(ctx, err, "loading packs for queries") } @@ -576,6 +580,10 @@ func (ds *Datastore) ListQueries(ctx context.Context, opt fleet.ListQueryOptions // loadPacksForQueries loads the user packs (aka 2017 packs) associated with the provided queries. func (ds *Datastore) loadPacksForQueries(ctx context.Context, queries []*fleet.Query) error { + return loadPacksForQueries(ctx, ds.reader(ctx), queries) +} + +func loadPacksForQueries(ctx context.Context, db sqlx.QueryerContext, queries []*fleet.Query) error { if len(queries) == 0 { return nil } @@ -609,7 +617,7 @@ func (ds *Datastore) loadPacksForQueries(ctx context.Context, queries []*fleet.Q fleet.Pack }{} - err = sqlx.SelectContext(ctx, ds.reader(ctx), &rows, query, args...) + err = sqlx.SelectContext(ctx, db, &rows, query, args...) if err != nil { return ctxerr.Wrap(ctx, err, "selecting load packs for queries") } diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 848eea200f2a..0abc88d950b6 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/fleetdm/fleet/v4/pkg/automatic_policy" "github.com/fleetdm/fleet/v4/server/authz" "github.com/fleetdm/fleet/v4/server/contexts/ctxerr" "github.com/fleetdm/fleet/v4/server/fleet" @@ -197,6 +198,12 @@ INSERT INTO software_installers ( return ctxerr.Wrap(ctx, err, "upsert software installer labels") } + if payload.AutomaticInstall { + if err := ds.createAutomaticPolicy(ctx, tx, payload, installerID); err != nil { + return ctxerr.Wrap(ctx, err, "create automatic policy") + } + } + return nil }); err != nil { return 0, 0, ctxerr.Wrap(ctx, err, "insert software installer") @@ -205,6 +212,55 @@ INSERT INTO software_installers ( return installerID, titleID, nil } +func (ds *Datastore) createAutomaticPolicy(ctx context.Context, tx sqlx.ExtContext, payload *fleet.UploadSoftwareInstallerPayload, softwareInstallerID uint) error { + generatedPolicyData, err := automatic_policy.Generate(automatic_policy.InstallerMetadata{ + Title: payload.Title, + Extension: payload.Extension, + BundleIdentifier: payload.BundleIdentifier, + PackageIDs: payload.PackageIDs, + }) + if err != nil { + return ctxerr.Wrap(ctx, err, "generate automatic policy query data") + } + teamID := fleet.PolicyNoTeamID + if payload.TeamID != nil { + teamID = *payload.TeamID + } + availablePolicyName, err := getAvailablePolicyName(ctx, tx, teamID, generatedPolicyData.Name) + if err != nil { + return ctxerr.Wrap(ctx, err, "get available policy name") + } + var userID *uint + if ctxUser := authz.UserFromContext(ctx); ctxUser != nil { + userID = &ctxUser.ID + } + if _, err := newTeamPolicy(ctx, tx, teamID, userID, fleet.PolicyPayload{ + Name: availablePolicyName, + Query: generatedPolicyData.Query, + Platform: generatedPolicyData.Platform, + Description: generatedPolicyData.Description, + SoftwareInstallerID: &softwareInstallerID, + }); err != nil { + return ctxerr.Wrap(ctx, err, "create automatic policy query") + } + return nil +} + +func getAvailablePolicyName(ctx context.Context, db sqlx.QueryerContext, teamID uint, tentativePolicyName string) (string, error) { + availableName := tentativePolicyName + for i := 2; ; i++ { + var count int + if err := sqlx.GetContext(ctx, db, &count, `SELECT COUNT(*) FROM policies WHERE team_id = ? AND name = ?`, teamID, availableName); err != nil { + return "", ctxerr.Wrapf(ctx, err, "get policy by team and name") + } + if count == 0 { + break + } + availableName = fmt.Sprintf("%s %d", tentativePolicyName, i) + } + return availableName, nil +} + func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, payload *fleet.UploadSoftwareInstallerPayload) (uint, error) { selectStmt := `SELECT id FROM software_titles WHERE name = ? AND source = ? AND browser = ''` selectArgs := []any{payload.Title, payload.Source} diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 72b714290842..56d996ebe59e 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -39,6 +39,7 @@ func TestSoftwareInstallers(t *testing.T) { {"GetHostLastInstallData", testGetHostLastInstallData}, {"GetOrGenerateSoftwareInstallerTitleID", testGetOrGenerateSoftwareInstallerTitleID}, {"BatchSetSoftwareInstallersScopedViaLabels", testBatchSetSoftwareInstallersScopedViaLabels}, + {"MatchOrCreateSoftwareInstallerWithAutomaticPolicies", testMatchOrCreateSoftwareInstallerWithAutomaticPolicies}, } for _, c := range cases { @@ -1776,3 +1777,233 @@ func testBatchSetSoftwareInstallersScopedViaLabels(t *testing.T, ds *Datastore) } } } + +func testMatchOrCreateSoftwareInstallerWithAutomaticPolicies(t *testing.T, ds *Datastore) { + ctx := context.Background() + + user1 := test.NewUser(t, ds, "Alice", "alice@example.com", true) + team1, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 1"}) + require.NoError(t, err) + team2, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 2"}) + require.NoError(t, err) + + tfr1, err := fleet.NewTempFileReader(strings.NewReader("hello"), t.TempDir) + require.NoError(t, err) + + // Test pkg without automatic install doesn't create policy. + _, _, err = ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + BundleIdentifier: "com.manual.foobar", + Extension: "pkg", + StorageID: "storage0", + Filename: "foobar0", + Title: "Manual foobar", + Version: "1.0", + Source: "apps", + UserID: user1.ID, + TeamID: &team1.ID, + AutomaticInstall: false, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team1Policies, _, err := ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Empty(t, team1Policies) + + // Test pkg. + installerID1, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + BundleIdentifier: "com.foo.bar", + Extension: "pkg", + StorageID: "storage1", + Filename: "foobar1", + Title: "Foobar", + Version: "1.0", + Source: "apps", + UserID: user1.ID, + TeamID: &team1.ID, + AutomaticInstall: true, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team1Policies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, team1Policies, 1) + require.Equal(t, "[Install software] Foobar (pkg)", team1Policies[0].Name) + require.Equal(t, "SELECT 1 FROM apps WHERE bundle_identifier = 'com.foo.bar';", team1Policies[0].Query) + require.Equal(t, "Policy triggers automatic install of Foobar on each host that's missing this software.", team1Policies[0].Description) + require.Equal(t, "darwin", team1Policies[0].Platform) + require.NotNil(t, team1Policies[0].SoftwareInstallerID) + require.Equal(t, installerID1, *team1Policies[0].SoftwareInstallerID) + require.NotNil(t, team1Policies[0].TeamID) + require.Equal(t, team1.ID, *team1Policies[0].TeamID) + + // Test msi. + installerID2, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + Extension: "msi", + StorageID: "storage2", + Filename: "zoobar1", + Title: "Zoobar", + Version: "1.0", + Source: "programs", + UserID: user1.ID, + TeamID: nil, + AutomaticInstall: true, + PackageIDs: []string{"id1"}, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + noTeamPolicies, _, err := ds.ListTeamPolicies(ctx, fleet.PolicyNoTeamID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, noTeamPolicies, 1) + require.Equal(t, "[Install software] Zoobar (msi)", noTeamPolicies[0].Name) + require.Equal(t, "SELECT 1 FROM programs WHERE identifying_number = 'id1';", noTeamPolicies[0].Query) + require.Equal(t, "Policy triggers automatic install of Zoobar on each host that's missing this software.", noTeamPolicies[0].Description) + require.Equal(t, "windows", noTeamPolicies[0].Platform) + require.NotNil(t, noTeamPolicies[0].SoftwareInstallerID) + require.Equal(t, installerID2, *noTeamPolicies[0].SoftwareInstallerID) + require.NotNil(t, noTeamPolicies[0].TeamID) + require.Equal(t, fleet.PolicyNoTeamID, *noTeamPolicies[0].TeamID) + + // Test deb. + installerID3, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + Extension: "deb", + StorageID: "storage3", + Filename: "barfoo1", + Title: "Barfoo", + Version: "1.0", + Source: "deb_packages", + UserID: user1.ID, + TeamID: &team2.ID, + AutomaticInstall: true, + PackageIDs: []string{"id1"}, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team2Policies, _, err := ds.ListTeamPolicies(ctx, team2.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, team2Policies, 1) + require.Equal(t, "[Install software] Barfoo (deb)", team2Policies[0].Name) + require.Equal(t, `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM deb_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM deb_packages WHERE name = 'Barfoo' +);`, team2Policies[0].Query) + require.Equal(t, `Policy triggers automatic install of Barfoo on each host that's missing this software. +Software won't be installed on Linux hosts with RPM-based distributions because this policy's query is written to always pass on these hosts.`, team2Policies[0].Description) + require.Equal(t, "linux", team2Policies[0].Platform) + require.NotNil(t, team2Policies[0].SoftwareInstallerID) + require.Equal(t, installerID3, *team2Policies[0].SoftwareInstallerID) + require.NotNil(t, team2Policies[0].TeamID) + require.Equal(t, team2.ID, *team2Policies[0].TeamID) + + // Test rpm. + installerID4, _, err := ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + Extension: "rpm", + StorageID: "storage4", + Filename: "barzoo1", + Title: "Barzoo", + Version: "1.0", + Source: "rpm_packages", + UserID: user1.ID, + TeamID: &team2.ID, + AutomaticInstall: true, + PackageIDs: []string{"id1"}, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team2Policies, _, err = ds.ListTeamPolicies(ctx, team2.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, team2Policies, 2) + require.Equal(t, "[Install software] Barzoo (rpm)", team2Policies[1].Name) + require.Equal(t, `SELECT 1 WHERE EXISTS ( + SELECT 1 WHERE (SELECT COUNT(*) FROM rpm_packages) = 0 +) OR EXISTS ( + SELECT 1 FROM rpm_packages WHERE name = 'Barzoo' +);`, team2Policies[1].Query) + require.Equal(t, `Policy triggers automatic install of Barzoo on each host that's missing this software. +Software won't be installed on Linux hosts with Debian-based distributions because this policy's query is written to always pass on these hosts.`, team2Policies[1].Description) + require.Equal(t, "linux", team2Policies[1].Platform) + require.NotNil(t, team2Policies[0].SoftwareInstallerID) + require.Equal(t, installerID4, *team2Policies[1].SoftwareInstallerID) + require.NotNil(t, team2Policies[1].TeamID) + require.Equal(t, team2.ID, *team2Policies[1].TeamID) + + _, err = ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, fleet.PolicyPayload{ + Name: "[Install software] OtherFoobar (pkg)", + Query: "SELECT 1;", + }) + require.NoError(t, err) + + // Test pkg and policy with name already exists. + _, _, err = ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + BundleIdentifier: "com.foo2.bar2", + Extension: "pkg", + StorageID: "storage5", + Filename: "foobar5", + Title: "OtherFoobar", + Version: "2.0", + Source: "apps", + UserID: user1.ID, + TeamID: &team1.ID, + AutomaticInstall: true, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team1Policies, _, err = ds.ListTeamPolicies(ctx, team1.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, team1Policies, 3) + require.Equal(t, "[Install software] OtherFoobar (pkg) 2", team1Policies[2].Name) + + team3, err := ds.NewTeam(ctx, &fleet.Team{Name: "team 3"}) + require.NoError(t, err) + + _, err = ds.NewTeamPolicy(ctx, team3.ID, &user1.ID, fleet.PolicyPayload{ + Name: "[Install software] Something2 (msi)", + Query: "SELECT 1;", + }) + require.NoError(t, err) + _, err = ds.NewTeamPolicy(ctx, team3.ID, &user1.ID, fleet.PolicyPayload{ + Name: "[Install software] Something2 (msi) 2", + Query: "SELECT 1;", + }) + require.NoError(t, err) + // This name is on another team, so it shouldn't count. + _, err = ds.NewTeamPolicy(ctx, team1.ID, &user1.ID, fleet.PolicyPayload{ + Name: "[Install software] Something2 (msi) 3", + Query: "SELECT 1;", + }) + require.NoError(t, err) + + // Test msi and policy with name already exists. + _, _, err = ds.MatchOrCreateSoftwareInstaller(ctx, &fleet.UploadSoftwareInstallerPayload{ + InstallerFile: tfr1, + Extension: "msi", + StorageID: "storage6", + Filename: "foobar6", + Title: "Something2", + PackageIDs: []string{"id2"}, + Version: "2.0", + Source: "programs", + UserID: user1.ID, + TeamID: &team3.ID, + AutomaticInstall: true, + ValidatedLabels: &fleet.LabelIdentsWithScope{}, + }) + require.NoError(t, err) + + team3Policies, _, err := ds.ListTeamPolicies(ctx, team3.ID, fleet.ListOptions{}, fleet.ListOptions{}) + require.NoError(t, err) + require.Len(t, team3Policies, 3) + require.Equal(t, "[Install software] Something2 (msi) 3", team3Policies[2].Name) +} diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index ad5c848818a1..fd108ce7b2b4 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -340,7 +340,8 @@ type UploadSoftwareInstallerPayload struct { LabelsExcludeAny []string // names of "exclude any" labels // ValidatedLabels is a struct that contains the validated labels for the software installer. It // is nil if the labels have not been validated. - ValidatedLabels *LabelIdentsWithScope + ValidatedLabels *LabelIdentsWithScope + AutomaticInstall bool } type UpdateSoftwareInstallerPayload struct { @@ -444,7 +445,7 @@ type SoftwarePackageOrApp struct { AppStoreID string `json:"app_store_id,omitempty"` // Name is only present for software installer packages. Name string `json:"name,omitempty"` - // AutomaticInstallPolicies is only present for Fleet maintained apps + // AutomaticInstallPolicies is present for Fleet maintained apps and custom packages // installed automatically with a policy. AutomaticInstallPolicies []AutomaticInstallPolicy `json:"automatic_install_policies"` diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index 73e47a147aa2..889e2b55115d 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -13318,6 +13318,37 @@ func (s *integrationEnterpriseTestSuite) TestPKGNoVersion() { s.uploadSoftwareInstaller(t, payload, http.StatusBadRequest, "Couldn't add. Fleet couldn't read the version from no_version.pkg.") } +func (s *integrationEnterpriseTestSuite) TestPKGNoBundleIdentifier() { + t := s.T() + ctx := context.Background() + + team, err := s.ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) + require.NoError(t, err) + + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some installer script", + Filename: "no_bundle_identifier.pkg", + TeamID: &team.ID, + } + s.uploadSoftwareInstaller(t, payload, http.StatusBadRequest, "Couldn't add. Unable to extract necessary metadata.") +} + +func (s *integrationEnterpriseTestSuite) TestAutomaticPoliciesWithExeFails() { + t := s.T() + ctx := context.Background() + + team, err := s.ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) + require.NoError(t, err) + + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some installer script", + Filename: "hello-world-installer.exe", + TeamID: &team.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, payload, http.StatusBadRequest, "Couldn't add. Fleet can't create a policy to detect existing installations for .exe packages. Please add the software, add a custom policy, and enable the install software policy automation.") +} + // 1. host reports software // 2. reconciler runs, creates title // 3. installer is uploaded, matches existing software title @@ -16369,3 +16400,104 @@ func (s *integrationEnterpriseTestSuite) TestListHostSoftwareWithLabelScoping() s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), nil, http.StatusOK, &getHostSw) require.Empty(t, getHostSw.Software) } + +func (s *integrationEnterpriseTestSuite) TestAutomaticPolicies() { + t := s.T() + ctx := context.Background() + + team1, err := s.ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team1"}) + require.NoError(t, err) + team2, err := s.ds.NewTeam(ctx, &fleet.Team{Name: t.Name() + "team2"}) + require.NoError(t, err) + + // Upload dummy_installer.pkg to team1 without automatic policy. + pkgPayload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some pkg install script", + Filename: "dummy_installer.pkg", + TeamID: &team1.ID, + } + s.uploadSoftwareInstaller(t, pkgPayload, http.StatusOK, "") + + // Check no policies were created. + ts := listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 0) + require.Len(t, ts.InheritedPolicies, 0) + + // Delete and try again with automatic policy turned on. + pkgTitleID := getSoftwareTitleID(t, s.ds, "DummyApp.app", "apps") + s.Do("DELETE", fmt.Sprintf("/api/latest/fleet/software/titles/%d/available_for_install", pkgTitleID), nil, http.StatusNoContent, + "team_id", fmt.Sprintf("%d", team1.ID)) + + // Upload dummy_installer.pkg to team1 with automatic policy. + pkgPayload = &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some pkg install script 2", + Filename: "dummy_installer.pkg", + TeamID: &team1.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, pkgPayload, http.StatusOK, "") + + pkgTitleID = getSoftwareTitleID(t, s.ds, "DummyApp.app", "apps") + respTitle := getSoftwareTitleResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/software/titles/%d?team_id=%d", pkgTitleID, team1.ID), listSoftwareTitlesRequest{}, http.StatusOK, &respTitle) + require.NotNil(t, respTitle.SoftwareTitle) + require.NotNil(t, respTitle.SoftwareTitle.SoftwarePackage) + require.Len(t, respTitle.SoftwareTitle.SoftwarePackage.AutomaticInstallPolicies, 1) + require.Equal(t, "[Install software] DummyApp.app (pkg)", respTitle.SoftwareTitle.SoftwarePackage.AutomaticInstallPolicies[0].Name) + + // Check a policy was created on team1. + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 1) + require.Len(t, ts.InheritedPolicies, 0) + require.Equal(t, "[Install software] DummyApp.app (pkg)", ts.Policies[0].Name) + + // Upload dummy_installer.pkg to team2 with automatic policy. + pkgPayload = &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "some pkg install script 3", + Filename: "dummy_installer.pkg", + TeamID: &team2.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, pkgPayload, http.StatusOK, "") + + // Check a policy was created on team2. + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team2.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 1) + require.Len(t, ts.InheritedPolicies, 0) + require.Equal(t, "[Install software] DummyApp.app (pkg)", ts.Policies[0].Name) + + // Upload ruby.deb to team1 with automatic policy. + payloadRubyDEB := &fleet.UploadSoftwareInstallerPayload{ + Filename: "ruby.deb", + TeamID: &team1.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, payloadRubyDEB, http.StatusOK, "") + + payloadRubyRPM := &fleet.UploadSoftwareInstallerPayload{ + Filename: "ruby.rpm", + TeamID: &team1.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, payloadRubyRPM, http.StatusOK, "") + + // Upload fleet-osquery.msi to team1 with automatic policy. + fleetOsqueryPayload := &fleet.UploadSoftwareInstallerPayload{ + Filename: "fleet-osquery.msi", + TeamID: &team1.ID, + AutomaticInstall: true, + } + s.uploadSoftwareInstaller(t, fleetOsqueryPayload, http.StatusOK, "") + + // Check policies were created on team1. + ts = listTeamPoliciesResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/teams/%d/policies", team1.ID), nil, http.StatusOK, &ts) + require.Len(t, ts.Policies, 4) + require.Len(t, ts.InheritedPolicies, 0) + require.Equal(t, "[Install software] ruby (deb)", ts.Policies[1].Name) + require.Equal(t, "[Install software] ruby (rpm)", ts.Policies[2].Name) + require.Equal(t, "[Install software] Fleet osquery (msi)", ts.Policies[3].Name) +} diff --git a/server/service/software_installers.go b/server/service/software_installers.go index a2b66dd5e7f3..fd96babceb54 100644 --- a/server/service/software_installers.go +++ b/server/service/software_installers.go @@ -32,6 +32,7 @@ type uploadSoftwareInstallerRequest struct { UninstallScript string LabelsIncludeAny []string LabelsExcludeAny []string + AutomaticInstall bool } type updateSoftwareInstallerRequest struct { @@ -291,7 +292,6 @@ func (uploadSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http decoded.SelfService = parsed } - // decode labels // decode labels var inclAny, exclAny []string var existsInclAny, existsExclAny bool @@ -316,6 +316,15 @@ func (uploadSoftwareInstallerRequest) DecodeRequest(ctx context.Context, r *http decoded.LabelsExcludeAny = exclAny } + val, ok = r.MultipartForm.Value["automatic_install"] + if ok && len(val) > 0 && val[0] != "" { + parsed, err := strconv.ParseBool(val[0]) + if err != nil { + return nil, &fleet.BadRequestError{Message: fmt.Sprintf("failed to decode automatic_install bool in multipart form: %s", err.Error())} + } + decoded.AutomaticInstall = parsed + } + return &decoded, nil } @@ -346,6 +355,7 @@ func uploadSoftwareInstallerEndpoint(ctx context.Context, request interface{}, s UninstallScript: req.UninstallScript, LabelsIncludeAny: req.LabelsIncludeAny, LabelsExcludeAny: req.LabelsExcludeAny, + AutomaticInstall: req.AutomaticInstall, } if err := svc.UploadSoftwareInstaller(ctx, payload); err != nil { diff --git a/server/service/testdata/software-installers/README.md b/server/service/testdata/software-installers/README.md index 7baa9592be02..63e84b5c0503 100644 --- a/server/service/testdata/software-installers/README.md +++ b/server/service/testdata/software-installers/README.md @@ -1,4 +1,5 @@ # testdata - `fleet-osquery.msi` is a dummy MSI installer created by `packaging.BuildMSI` with a fake `orbit.exe` that just has `hello world` in it. Its software title is `Fleet osquery` and its version is `1.0.0`. -- `ruby.rpm` was downloaded from https://rpmfind.net/linux/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/r/ruby-3.3.5-15.fc42.x86_64.rpm. \ No newline at end of file +- `ruby.rpm` was downloaded from https://rpmfind.net/linux/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/r/ruby-3.3.5-15.fc42.x86_64.rpm. +- `no_bundle_identifier.pkg` was generated with the following command `pkgbuild --nopayload --install-location "/" --scripts scripts/ --identifier ' ' --version '1.0.0' no_bundle_identifier.pkg` (where `scripts/` contained a dummy `preinstall` and `postinstall` scripts). \ No newline at end of file diff --git a/server/service/testdata/software-installers/no_bundle_identifier.pkg b/server/service/testdata/software-installers/no_bundle_identifier.pkg new file mode 100644 index 000000000000..2137b85278cf Binary files /dev/null and b/server/service/testdata/software-installers/no_bundle_identifier.pkg differ diff --git a/server/service/testing_client.go b/server/service/testing_client.go index a8f2183d1aa6..9c6ce0df8f16 100644 --- a/server/service/testing_client.go +++ b/server/service/testing_client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "mime/multipart" @@ -560,6 +561,14 @@ func (ts *withServer) uploadSoftwareInstaller( t.Helper() tfr, err := fleet.NewKeepFileReader(filepath.Join("testdata", "software-installers", payload.Filename)) + // Try the test installers in the pkg/file testdata (to reduce clutter/copies). + if errors.Is(err, os.ErrNotExist) { + var err2 error + tfr, err2 = fleet.NewKeepFileReader(filepath.Join("..", "..", "pkg", "file", "testdata", "software-installers", payload.Filename)) + if err2 == nil { + err = nil + } + } require.NoError(t, err) defer tfr.Close() @@ -597,6 +606,9 @@ func (ts *withServer) uploadSoftwareInstaller( require.NoError(t, w.WriteField("labels_exclude_any", l)) } } + if payload.AutomaticInstall { + require.NoError(t, w.WriteField("automatic_install", "true")) + } w.Close() diff --git a/tools/custom-package-parser/README.md b/tools/custom-package-parser/README.md new file mode 100644 index 000000000000..88371a25c590 --- /dev/null +++ b/tools/custom-package-parser/README.md @@ -0,0 +1,20 @@ +# custom-package-parser + +Tool to extract the metadata of software packages (same way Fleet would extract metadata on uploads). +This tool was used to determine accuracy of Fleet's processing of software packages (with the most used/popular apps) (see [tests.md](./tests.md)). + +Using a local file: +```sh +go run ./tools/custom-package-parser -path ~/Downloads/MicrosoftTeams.pkg +- Name: 'Microsoft Teams.app' +- Bundle Identifier: 'com.microsoft.teams2' +- Package IDs: 'com.microsoft.teams2,com.microsoft.package.Microsoft_AutoUpdate.app,com.microsoft.MSTeamsAudioDevice' +``` + +Using a URL: +```sh +go run ./tools/custom-package-parser -url https://downloads.1password.com/win/1PasswordSetup-latest.msi +- Name: '1Password' +- Bundle Identifier: '' +- Package IDs: '{321BD799-2490-40D7-8A88-6888809FA681}' +``` \ No newline at end of file diff --git a/tools/custom-package-parser/main.go b/tools/custom-package-parser/main.go new file mode 100644 index 000000000000..d64506625e2c --- /dev/null +++ b/tools/custom-package-parser/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "strings" + + "github.com/fleetdm/fleet/v4/pkg/file" + "github.com/fleetdm/fleet/v4/pkg/fleethttp" + "github.com/fleetdm/fleet/v4/server/fleet" +) + +func main() { + url := flag.String("url", "", "URL of the custom package") + path := flag.String("path", "", "File path of the custom package") + flag.Parse() + + if *url == "" && *path == "" { + log.Fatal("missing -url or -path argument") + } + if *url != "" && *path != "" { + log.Fatal("cannot set both -url and -path") + } + + metadata, err := processPackage(*url, *path) + if err != nil { + log.Fatal(err) + } + + fmt.Printf( + "- Name: '%s'\n- Bundle Identifier: '%s'\n- Package IDs: '%s'\n", + metadata.Name, metadata.BundleIdentifier, strings.Join(metadata.PackageIDs, ","), + ) +} + +func processPackage(url, path string) (*file.InstallerMetadata, error) { + var tfr *fleet.TempFileReader + if url != "" { + client := fleethttp.NewClient() + client.Transport = fleethttp.NewSizeLimitTransport(fleet.MaxSoftwareInstallerSize) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("create http request: %s", err) + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("get request: %s", err) + } + defer resp.Body.Close() + + // Allow all 2xx and 3xx status codes in this pass. + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("get request failed with status: %d", resp.StatusCode) + } + + tfr, err = fleet.NewTempFileReader(resp.Body, nil) + if err != nil { + return nil, fmt.Errorf("reading custom package: %d", resp.StatusCode) + } + defer tfr.Close() + } else { // -path + fp, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open file: %s", err) + } + tfr = &fleet.TempFileReader{ + File: fp, + } + } + + metadata, err := file.ExtractInstallerMetadata(tfr) + if err != nil { + return nil, fmt.Errorf("extract installer metadata: %s", err) + } + return metadata, nil +} diff --git a/tools/custom-package-parser/tests.md b/tools/custom-package-parser/tests.md new file mode 100644 index 000000000000..5f9fa4da7876 --- /dev/null +++ b/tools/custom-package-parser/tests.md @@ -0,0 +1,564 @@ +# Custom packages tests + +This document aims to provide us with some stats for the most used apps with respect to what Fleet extracts from the installers and what osquery reports for the installed applications. +The goal is to improve the accuracy of automatically generated policy queries for installers. + +## Results + +The results have been calculated using many of the apps in the current list of FMA apps for macOS (as of December 2024). + +### pkg + +11 `pkg`s were tested: +- Matching of extracted bundle identifier and osquery's reported `bundle_identifier`: 100% (11/11) +- Matching of extracted package title/name and osquery's reported `apps.name`: 100% (11/11) + +### msi + +10 `msi`s were tested: +- Matching of extracted GUID and osquery's reported `programs.identifying_number`: 90% (9/10) +- Matching of extracted package title/name and osquery's reported `programs.name`: 90% (9/10) + +### exe + +13 `exe`s were tested: +- Matching of extracted package title/name and osquery's reported `programs.name`: ~30% (4/13) + +### deb + +6 `deb`s were tested: +- Matching of extracted package title/name and osquery's reported `deb_packages.name`: ~100% (6/6) + +### rpm + +6 `rpm`s were tested: +- Matching of extracted package title/name and osquery's reported `deb_packages.name`: ~100% (6/6) + +## Tests + +### 1Password + +#### pkg + +✅ https://downloads.1password.com/mac/1Password.pkg +- Bundle Identifier: 'com.1password.1password' +- Name: '1Password.app' (matches osquery's apps.name) +- Package IDs: 'com.1password.1password' + +#### exe + +✅ https://downloads.1password.com/win/1PasswordSetup-latest.exe +- Default installer script didn't work. +- Running `1PasswordSetup-latest.exe --silent` on the `cmd` works, but not via Fleet because the installer is per-user, whereas the MSI is system-wide, see https://support.1password.com/deploy-1password/. +Extracted metadata: +- Name: '1Password' (matches osquery's `programs.name`) +- Package IDs: '1Password' + +#### msi + +✅ https://downloads.1password.com/win/1PasswordSetup-latest.msi +- Name: '1Password' (matches osquery's `programs.name`) +- Package IDs: '{321BD799-2490-40D7-8A88-6888809FA681}' (matches osquery's `programs.identifying_number`) + +#### deb + +✅ https://downloads.1password.com/linux/debian/amd64/stable/1password-latest.deb +- Name: '1password' (matches osquery's `deb_packages.name`) +- Package IDs: '1password' + +#### rpm + +✅ https://downloads.1password.com/linux/rpm/stable/x86_64/1password-latest.rpm +- Name: '1password' (matches osquery's `rpm_packages.name`) +- Package IDs: '1password' + +### Adobe Acrobat Reader + +#### pkg + +N/A (they have .dmg) + +#### exe + +❌ https://ardownload2.adobe.com/pub/adobe/acrobat/win/AcrobatDC/2400520320/AcroRdrDCx642400520320_en_US.exe +- Name: 'Adobe Self Extractor' (osquery reports `Adobe Acrobat (64-bit)` in `programs.name`) +- Package IDs: 'Adobe Self Extractor' + +#### msi + +N/A + +#### deb + +N/A + +#### rpm + +N/A + +### Box Drive + +#### pkg + +✅ https://e3.boxcdn.net/desktop/releases/mac/BoxDrive.pkg +- Name: 'Box.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.box.desktop' +- Package IDs: 'com.box.desktop.installer.autoupdater,com.box.desktop.installer.desktop,com.box.desktop.installer.local.appsupport' + +#### msi + +✅ https://e3.boxcdn.net/desktop/releases/win/BoxDrive.msi +- Name: 'Box' (matches osquery's `programs.name`) +- Package IDs: '{9ACD1AAB-DCE9-480D-A7A4-5470D5E4E10F}' (matches osquery's `programs.identifying_number`) + +#### exe + +N/A + +#### deb + +N/A + +#### rpm + +N/A + +### Brave Browser + +#### pkg + +✅ https://github.com/brave/brave-browser/releases/download/v1.73.101/Brave-Browser-universal.pkg +- Name: 'Brave Browser.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.brave.Browser' +- Package IDs: 'com.brave.Browser,com.brave.Browser.helper.renderer,com.brave.Updater,com.brave.Keystone,com.brave.Browser.framework,com.brave.Browser.helper,com.brave.Browser.helper.plugin,org.sparkle-project.Sparkle,org.sparkle-project.Sparkle.Autoupdate,com.brave.Keystone.Agent,com.brave.Browser.framework.AlertNotificationService' + +#### exe + +❌ https://referrals.brave.com/latest/BraveBrowserSetup.exe +- Default installer script doesn't work. +- Name: 'BraveSoftware Update' (does not match osquery's `programs.name`, which is 'Brave') +- Package IDs: 'BraveSoftware Update' + +#### msi + +N/A + +#### deb + +✅ https://github.com/brave/brave-browser/releases/download/v1.73.101/brave-browser_1.73.101_amd64.deb +- Default installer script doesn't work. +- Name: 'brave-browser' (matches osquery's `deb_packages.name`) +- Package IDs: 'brave-browser' + +#### rpm + +✅ https://github.com/brave/brave-browser/releases/download/v1.73.101/brave-browser-1.73.101-1.x86_64.rpm +- Default installer script doesn't work. +- Name: 'brave-browser' (matches osquery's `rpm_packages.name`) +- Package IDs: 'brave-browser' + +### Cloudflare WARP + +#### pkg + +✅ https://appcenter-filemanagement-distrib5ede6f06e.azureedge.net/e638644a-02a2-4a21-aa30-8a9a1bf774ce/Cloudflare_WARP_2024.11.309.0.pkg +- Name: 'Cloudflare WARP.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.cloudflare.1dot1dot1dot1.macos' +- Package IDs: 'com.cloudflare.1dot1dot1dot1.macos' + +#### msi + +✅ https://appcenter-filemanagement-distrib3ede6f06e.azureedge.net/679d20da-1684-49df-89e5-e976ec1c010c/Cloudflare_WARP_2024.11.309.0.msi +- Name: 'Cloudflare WARP' (matches osquery's `programs.name`) +- Package IDs: '{2BC6DCCB-7E9D-44D7-A525-6F6C6E83C419}' (matches osquery's `programs.identifying_number`) + +#### exe + +N/A + +#### deb + +✅ https://pkg.cloudflareclient.com/pool/focal/main/c/cloudflare-warp/cloudflare-warp_2024.11.309.0_amd64.deb +- Name: 'cloudflare-warp' (matches osquery's `deb_packages.name`) +- Package IDs: 'cloudflare-warp' + +#### rpm + +N/A + +### Docker + +#### pkg + +N/A (has dmg, pkg requires admin account in app.docker.com) + +#### msi + +N/A (msi requires admin account in app.docker.com) + +#### exe + +❌ https://desktop.docker.com/win/main/amd64/Docker%20Desktop%20Installer.exe +- Default installer script doesn't work. +- Name: 'Docker Desktop Installer' (doesn't match osquery's `programs.name`) +- Package IDs: 'Docker Desktop Installer' + +#### deb + +✅ https://desktop.docker.com/linux/main/amd64/docker-desktop-amd64.deb +- Name: 'docker-desktop' (matches osquery's `deb_packages.name`) +- Package IDs: 'docker-desktop' + +#### rpm + +❌ https://desktop.docker.com/linux/main/amd64/docker-desktop-x86_64.rpm +- Default installer script doesn't work on my Fedora 38 VM. + +### Figma + +#### pkg + +✅ https://desktop.figma.com/mac-universal/Figma-124.6.5.pkg +- Name: 'Figma.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.figma.Desktop' +- Package IDs: 'com.figma.Desktop' + +#### msi + +✅ https://desktop.figma.com/win/Figma-124.6.5.msi +- Name: 'Figma (Machine - MSI)' (matches osquery's `programs.name`) +- Package IDs: '{6332AF99-9139-41D1-98FC-BA21B9D6DE2E}' (matches osquery's `programs.identifying_number`) + +#### exe + +❌ https://desktop.figma.com/win/FigmaSetup.exe +- Default installer script doesn't work. +- Name: 'Figma Desktop' (doesnt match osquery's `programs.name`) +- Package IDs: 'Figma Desktop' + +#### deb + +✅ https://github.com/Figma-Linux/figma-linux/releases/download/v0.11.5/figma-linux_0.11.5_linux_amd64.deb +- Name: 'figma-linux' +- Package IDs: 'figma-linux' + +#### rpm + +✅ https://github.com/Figma-Linux/figma-linux/releases/download/v0.11.5/figma-linux_0.11.5_linux_x86_64.rpm +- Name: 'figma-linux' +- Package IDs: 'figma-linux' + +### Firefox + +#### pkg + +✅ https://ftp.mozilla.org/pub/firefox/releases/129.0.2/mac/en-US/Firefox%20129.0.2.pkg +- Name: 'Firefox.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'org.mozilla.firefox' +- Package IDs: 'org.mozilla.firefox' + +#### msi + +❌ https://ftp.mozilla.org/pub/firefox/releases/129.0.2/win64/en-US/Firefox%20Setup%20129.0.2.msi +- Name: 'Mozilla Firefox 129.0.2 x64 en-US' (doesn't match osquery's `programs.name`, `Mozilla Firefox (x64 en-US)`) +- Package IDs: '{1294A4C5-9977-480F-9497-C0EA1E630130}' (osquery returns empty `programs.identifying_number`) +- Default uninstall script doesn't work because it seems the installer doesn't set the GUID on the system registry. + +#### exe + +❌ https://download-installer.cdn.mozilla.net/pub/firefox/releases/133.0.3/win32/en-US/Firefox%20Installer.exe +- Default installer script succeeds but doesn't install Firefox +- Name: 'Firefox' (doesn't match osquery's `programs.name`, `Mozilla Firefox (x64 en-US`) +- Package IDs: 'Firefox' + +#### deb + +✅ https://ftp.mozilla.org/pub/firefox/releases/129.0.2/linux-x86_64/en-US/firefox-129.0.2.deb +- Name: 'firefox' (matches osquery's `deb_packages.name`) +- Package IDs: 'firefox' + +#### rpm + +Skipped. + +### Chrome + +#### pkg + +✅ https://dl.google.com/dl/chrome/mac/universal/stable/gcem/GoogleChrome.pkg +- Name: 'Google Chrome.app' (matches osquery's apps.name) +- Bundle Identifier: 'com.google.Chrome' +- Package IDs: 'com.google.Chrome' + +#### msi + +✅ https://dl.google.com/tag/s/appguid%3D%7B8A69D345-D564-463C-AFF1-A69D9E530F96%7D%26iid%3D%7BDAD35779-DEEF-9D60-7F91-7A3EEC3B65A9%7D%26lang%3Den%26browser%3D4%26usagestats%3D0%26appname%3DGoogle%2520Chrome%26needsadmin%3Dtrue%26ap%3Dx64-stable-statsdef_0%26brand%3DGCEA/dl/chrome/install/googlechromestandaloneenterprise64.msi +- Name: 'Google Chrome' (matches osquery's `programs.name`) +- Package IDs: '{D9596C6B-431E-3638-ACB7-B4B0D24D2D1B}' (matches osquery's `programs.identifying_number`) + +#### exe + +❌ https://dl.google.com/tag/s/appguid%3D%7B8A69D345-D564-463C-AFF1-A69D9E530F96%7D%26iid%3D%7B8CCBCFA1-CE41-77DB-B8C4-98742A89BC8D%7D%26lang%3Des-419%26browser%3D5%26usagestats%3D1%26appname%3DGoogle%2520Chrome%26needsadmin%3Dprefers%26ap%3Dx64-statsdef_1%26brand%3DUEAD%26installdataindex%3Dempty/update2/installers/ChromeSetup.exe +- Name: 'Google Installer' (doesn't match osquery's `programs.name`) +- Package IDs: 'Google Installer' + +#### deb + +Skipped. + +#### rpm + +✅ https://dl.google.com/linux/chrome/rpm/stable/x86_64/google-chrome-stable-129.0.6668.70-1.x86_64.rpm +- Name: 'google-chrome-stable' (matches osquery's `rpm_packages.name`) +- Package IDs: 'google-chrome-stable' + +### Microsoft Edge + +#### pkg + +✅ https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/8613322a-2386-49ce-a73f-0b718af56cfe/MicrosoftEdge-131.0.2903.99.pkg +- Name: 'Microsoft Edge.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.microsoft.edgemac' +- Package IDs: 'com.microsoft.edgemac' + +#### msi + +✅ https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/249fe233-1b7c-4b8d-93bc-e64ba81a0c02/MicrosoftEdgeEnterpriseX64.msi +- Name: 'Microsoft Edge' (matches osquery's `programs.name`) +- Package IDs: '{5DFDE950-0D8C-30AC-966B-EED2E340F09B}' (matches osquery's `programs.identifying_number`) + +#### exe + +N/A + +#### deb + +✅ https://packages.microsoft.com/repos/edge/pool/main/m/microsoft-edge-stable/microsoft-edge-stable_131.0.2903.99-1_amd64.deb +- Name: 'microsoft-edge-stable' (matches osquery's `deb_packages.name`) +- Package IDs: 'microsoft-edge-stable' + +#### rpm + +✅ https://packages.microsoft.com/yumrepos/edge/microsoft-edge-stable-131.0.2903.99-1.x86_64.rpm +- Name: 'microsoft-edge-stable' +- Package IDs: 'microsoft-edge-stable' + +### Microsoft Excel + +Skipped (not easy to get ahold of installers) + +### Microsoft Teams + +#### pkg + +✅ https://statics.teams.cdn.office.net/production-osx/enterprise/webview2/lkg/MicrosoftTeams.pkg +- Name: 'Microsoft Teams.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.microsoft.teams2' +- Package IDs: 'com.microsoft.MSTeamsAudioDevice,com.microsoft.teams2,com.microsoft.package.Microsoft_AutoUpdate.app' + +#### msi + +✅ https://statics.teams.cdn.office.net/production-windows-x64/1.7.00.33761/Teams_windows_x64.msi +- Default installer script doesn't work. +- Name: 'Teams Machine-Wide Installer' (matches osquery's `programs.name`) +- Package IDs: '{731F6BAA-A986-45A4-8936-7C3AAAAA760B}' (matches osquery's `programs.identifying_number`) + +#### exe + +❌ https://statics.teams.cdn.office.net/evergreen-assets/DesktopClient/MSTeamsSetup.exe +- Name: 'Microsoft Teams' (osquery does not return the entry for the installed Microsoft Teams on this setup, maybe a osquery bug?) +- Package IDs: 'Microsoft Teams' + +#### deb + +Skipped. + +#### rpm + +Skipped. + +### Microsoft Word + +Skipped (not easy to get ahold of installers) + +### Notion + +#### pkg + +N/A + +#### msi + +N/A + +#### exe + +✅ https://desktop-release.notion-static.com/Notion%20Setup%204.2.0.exe +- Name: 'Notion 4.2.0' (matches osquery's `programs.name`) +- Package IDs: 'Notion' + +#### deb + +Skipped. + +#### rpm + +Skipped. + +### Postman + +#### pkg + +N/A (they have a zip:app) + +#### msi + +N/A + +#### exe + +✅ https://dl.pstmn.io/download/latest/win64 +- Name: 'Postman' (matches osquery's `programs.name`) +- Package IDs: 'Postman' + +#### deb + +N/A (installer is just a tar.gz) + +#### rpm + +N/A (installer is just a tar.gz) + +### Slack + +#### pkg + +✅ https://downloads.slack-edge.com/desktop-releases/mac/x64/4.41.105/Slack-4.41.105-macOS.pkg +- Name: 'Slack.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'com.tinyspeck.slackmacgap' +- Package IDs: 'com.tinyspeck.slackmacgap' + +#### msi + +✅ https://downloads.slack-edge.com/desktop-releases/windows/x64/4.41.105/slack-standalone-4.41.105.0.msi +- Name: 'Slack (Machine - MSI)' (matches osquery's `programs.name`) +- Package IDs: '{D1458C20-B783-4E0C-B9D9-FAC9F56F94DB}' (matches osquery's `programs.identifying_number`) + +#### exe + +❌ https://downloads.slack-edge.com/desktop-releases/windows/x64/4.41.105/SlackSetup.exe +- Name: 'Slack Desktop' (doesn't match osquery's `programs.name`, `Slack`) +- Package IDs: 'Slack Desktop' + +#### deb + +Skipped. + +#### rpm + +✅ https://downloads.slack-edge.com/desktop-releases/linux/x64/4.39.95/slack-4.39.95-0.1.el8.x86_64.rpm +- Name: 'slack' (matches osquery's `rpm_packages.name`) +- Package IDs: 'slack' + +### Team Viewer + +#### pkg + +N/A (needs an admin license) + +#### msi + +N/A (needs an admin license) + +#### exe + +N/A (their exes are executables, not installers) + +#### deb + +Skipped. + +#### rpm + +Skipped. + +### Visual Studio Code + +#### pkg + +N/A + +#### msi + +N/A + +#### exe + +❌ https://vscode.download.prss.microsoft.com/dbazure/download/stable/fabdb6a30b49f79a7aba0f2ad9df9b399473380f/VSCodeSetup-x64-1.96.2.exe +- Name: 'Visual Studio Code' (doesn't match osquery's `programs.name`, `Microsoft Visual Studio Code`) +- Package IDs: 'Visual Studio Code' + +#### deb + +Skipped. + +#### rpm + +Skipped. + +### WhatsApp + +#### pkg + +N/A (they have a zip:app) + +#### msi + +N/A (from app store) + +#### exe + +N/A (from app store) + +#### deb + +N/A + +#### rpm + +N/A + +### Zoom for IT admins + +#### pkg + +✅ https://cdn.zoom.us/prod/6.3.0.44805/ZoomInstallerIT.pkg +- Name: 'zoom.us.app' (matches osquery's `apps.name`) +- Bundle Identifier: 'us.zoom.xos' +- Package IDs: 'us.zoom.pkg.videomeeting' + +#### msi + +✅ https://cdn.zoom.us/prod/6.3.0.52884/x64/ZoomInstallerFull.msi +- Name: 'Zoom Workplace (64-bit)' (matches osquery's `programs.name`) +- Package IDs: '{9BF959AB-C61A-460F-BA37-7D3DABB1388B}' (matches osquery's `programs.identifying_number`) + +#### exe + +Skipped. + +#### deb + +Skipped. + +#### rpm + +Skipped. + +### Tailscale + +#### exe + +✅ https://dl.tailscale.com/stable/tailscale-setup-1.72.0.exe +- Name: 'Tailscale' (matches osquery's `programs.name`). +- Package IDs: 'Tailscale'