From 468a5b818f4903e54a98c7ee0c1e001b5967aed8 Mon Sep 17 00:00:00 2001 From: Lucas Manuel Rodriguez Date: Fri, 15 Nov 2024 19:41:46 -0300 Subject: [PATCH] Cherry pick PR #23766 into RC v4.60.0 (#23865) Cherry pick PR #23766 into RC v4.60.0 Co-authored-by: Tim Lee Co-authored-by: Ian Littman --- changes/8750-add-team_identifier-to-software | 1 + .../Contributing/Understanding-host-vitals.md | 42 ++++--- orbit/changes/add-codesign-table | 1 + orbit/pkg/table/codesign/codesign_darwin.go | 100 +++++++++++++++++ orbit/pkg/table/extension_darwin.go | 3 + schema/osquery_fleet_schema.json | 25 +++++ schema/tables/codesign.yml | 15 +++ ...mIdentifierToHostSoftwareInstalledPaths.go | 23 ++++ server/datastore/mysql/schema.sql | 7 +- server/datastore/mysql/software.go | 70 ++++++++---- server/datastore/mysql/software_test.go | 22 ++-- server/fleet/datastore.go | 5 +- server/fleet/hosts.go | 3 + server/fleet/software.go | 19 +++- server/fleet/software_installer.go | 19 ++-- server/service/integration_core_test.go | 105 ++++++++++++++++++ server/service/osquery.go | 10 +- server/service/osquery_test.go | 1 + server/service/osquery_utils/queries.go | 81 +++++++++----- server/service/osquery_utils/queries_test.go | 4 +- 20 files changed, 455 insertions(+), 101 deletions(-) create mode 100644 changes/8750-add-team_identifier-to-software create mode 100644 orbit/changes/add-codesign-table create mode 100644 orbit/pkg/table/codesign/codesign_darwin.go create mode 100644 schema/tables/codesign.yml create mode 100644 server/datastore/mysql/migrations/tables/20241110152839_AddTeamIdentifierToHostSoftwareInstalledPaths.go diff --git a/changes/8750-add-team_identifier-to-software b/changes/8750-add-team_identifier-to-software new file mode 100644 index 000000000000..0d05d81b0944 --- /dev/null +++ b/changes/8750-add-team_identifier-to-software @@ -0,0 +1 @@ +* Added `team_identifier` signature information to Apple macOS applications to the `/api/latest/fleet/hosts/:id/software` API endpoint. diff --git a/docs/Contributing/Understanding-host-vitals.md b/docs/Contributing/Understanding-host-vitals.md index c1c6d2b0c9d1..aa3670ee63be 100644 --- a/docs/Contributing/Understanding-host-vitals.md +++ b/docs/Contributing/Understanding-host-vitals.md @@ -480,7 +480,6 @@ SELECT version AS version, identifier AS extension_id, browser_type AS browser, - 'Browser plugin (Chrome)' AS type, 'chrome_extensions' AS source, '' AS vendor, '' AS installed_path @@ -500,7 +499,6 @@ WITH cached_users AS (WITH cached_groups AS (select * from groups) SELECT name AS name, version AS version, - 'Package (deb)' AS type, '' AS extension_id, '' AS browser, 'deb_packages' AS source, @@ -514,7 +512,6 @@ UNION SELECT package AS name, version AS version, - 'Package (Portage)' AS type, '' AS extension_id, '' AS browser, 'portage_packages' AS source, @@ -527,7 +524,6 @@ UNION SELECT name AS name, version AS version, - 'Package (RPM)' AS type, '' AS extension_id, '' AS browser, 'rpm_packages' AS source, @@ -540,7 +536,6 @@ UNION SELECT name AS name, version AS version, - 'Package (NPM)' AS type, '' AS extension_id, '' AS browser, 'npm_packages' AS source, @@ -553,7 +548,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, identifier AS extension_id, browser_type AS browser, 'chrome_extensions' AS source, @@ -566,7 +560,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, identifier AS extension_id, 'firefox' AS browser, 'firefox_addons' AS source, @@ -579,7 +572,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS extension_id, '' AS browser, 'python_packages' AS source, @@ -603,7 +595,6 @@ WITH cached_users AS (WITH cached_groups AS (select * from groups) SELECT name AS name, COALESCE(NULLIF(bundle_short_version, ''), bundle_version) AS version, - 'Application (macOS)' AS type, bundle_identifier AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -616,7 +607,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -629,7 +619,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, '' AS bundle_identifier, identifier AS extension_id, browser_type AS browser, @@ -642,7 +631,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, '' AS bundle_identifier, identifier AS extension_id, 'firefox' AS browser, @@ -655,7 +643,6 @@ UNION SELECT name As name, version AS version, - 'Browser plugin (Safari)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -668,7 +655,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Homebrew)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -679,9 +665,27 @@ SELECT FROM homebrew_packages; ``` +## software_macos_codesign + +- Description: A software override query[^1] to append codesign information to macOS software entries. Requires `fleetd` + +- Platforms: darwin + +- Discovery query: +```sql +SELECT 1 FROM osquery_registry WHERE active = true AND registry = 'table' AND name = 'codesign' +``` + +- Query: +```sql +SELECT a.path, c.team_identifier + FROM apps a + JOIN codesign c ON a.path = c.path +``` + ## software_macos_firefox -- Description: A software override query[^1] to differentiate between Firefox and Firefox ESR on macOS. Requires `fleetd` +- Description: A software override query[^1] to differentiate between Firefox and Firefox ESR on macOS. Requires `fleetd` - Platforms: darwin @@ -709,7 +713,6 @@ WITH app_paths AS ( ELSE 'Firefox.app' END AS name, COALESCE(NULLIF(apps.bundle_short_version, ''), apps.bundle_version) AS version, - 'Application (macOS)' AS type, apps.bundle_identifier AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -740,7 +743,6 @@ WITH cached_users AS (WITH cached_groups AS (select * from groups) SELECT name, version, - 'IDE extension (VS Code)' AS type, '' AS bundle_identifier, uuid AS extension_id, '' AS browser, @@ -764,7 +766,6 @@ WITH cached_users AS (WITH cached_groups AS (select * from groups) SELECT name AS name, version AS version, - 'Program (Windows)' AS type, '' AS extension_id, '' AS browser, 'programs' AS source, @@ -775,7 +776,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS extension_id, '' AS browser, 'python_packages' AS source, @@ -786,7 +786,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (IE)' AS type, '' AS extension_id, '' AS browser, 'ie_extensions' AS source, @@ -797,7 +796,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, identifier AS extension_id, browser_type AS browser, 'chrome_extensions' AS source, @@ -808,7 +806,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, identifier AS extension_id, 'firefox' AS browser, 'firefox_addons' AS source, @@ -819,7 +816,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Chocolatey)' AS type, '' AS extension_id, '' AS browser, 'chocolatey_packages' AS source, diff --git a/orbit/changes/add-codesign-table b/orbit/changes/add-codesign-table new file mode 100644 index 000000000000..49b38025d636 --- /dev/null +++ b/orbit/changes/add-codesign-table @@ -0,0 +1 @@ +* Added `codesign` table to provide the "Team identifier" of macOS applications. diff --git a/orbit/pkg/table/codesign/codesign_darwin.go b/orbit/pkg/table/codesign/codesign_darwin.go new file mode 100644 index 000000000000..e1e8b26c9abd --- /dev/null +++ b/orbit/pkg/table/codesign/codesign_darwin.go @@ -0,0 +1,100 @@ +//go:build darwin +// +build darwin + +// Package codesign implements an extension osquery table +// to get signature information of macOS applications. +package codesign + +import ( + "bufio" + "bytes" + "context" + "errors" + "os/exec" + "strings" + + "github.com/osquery/osquery-go/plugin/table" + "github.com/rs/zerolog/log" +) + +// Columns is the schema of the table. +func Columns() []table.ColumnDefinition { + return []table.ColumnDefinition{ + // path is the absolute path to the app bundle. + // It's required and only supports the equality operator. + table.TextColumn("path"), + // team_identifier is the "Team ID", aka "Signature ID", "Developer ID". + // The value is "" if the app doesn't have a team identifier set. + // (this is the case for example for builtin Apple apps). + // + // See https://developer.apple.com/help/account/manage-your-team/locate-your-team-id/. + table.TextColumn("team_identifier"), + } +} + +// Generate is called to return the results for the table at query time. +// +// Constraints for generating can be retrieved from the queryContext. +func Generate(ctx context.Context, queryContext table.QueryContext) ([]map[string]string, error) { + constraints, ok := queryContext.Constraints["path"] + if !ok || len(constraints.Constraints) == 0 { + return nil, errors.New("missing path") + } + + var paths []string + for _, constraint := range constraints.Constraints { + if constraint.Operator != table.OperatorEquals { + return nil, errors.New("only supported operator for 'path' is '='") + } + paths = append(paths, constraint.Expression) + } + + var rows []map[string]string + for _, path := range paths { + row := map[string]string{ + "path": path, + "team_identifier": "", + } + output, err := exec.CommandContext(ctx, "/usr/bin/codesign", + // `codesign --display` does not perform any verification of executables/resources, + // it just parses and displays signature information read from the `Contents` folder. + "--display", + // If we don't set verbose it only prints the executable path. + "--verbose", + path, + ).CombinedOutput() // using CombinedOutput because output is in stderr and stdout is empty. + if err != nil { + // Logging as debug to prevent non signed apps to generate a lot of logged errors. + log.Debug().Err(err).Str("output", string(output)).Str("path", path).Msg("codesign --display failed") + rows = append(rows, row) + continue + } + info := parseCodesignOutput(output) + row["team_identifier"] = info.teamIdentifier + rows = append(rows, row) + } + + return rows, nil +} + +type parsedInfo struct { + teamIdentifier string +} + +func parseCodesignOutput(output []byte) parsedInfo { + const teamIdentifierPrefix = "TeamIdentifier=" + + scanner := bufio.NewScanner(bytes.NewReader(output)) + var info parsedInfo + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, teamIdentifierPrefix) { + info.teamIdentifier = strings.TrimSpace(strings.TrimPrefix(line, teamIdentifierPrefix)) + // "not set" is usually displayed on Apple builtin apps. + if info.teamIdentifier == "not set" { + info.teamIdentifier = "" + } + } + } + return info +} diff --git a/orbit/pkg/table/extension_darwin.go b/orbit/pkg/table/extension_darwin.go index 18bdc6884f5f..59f0b240077f 100644 --- a/orbit/pkg/table/extension_darwin.go +++ b/orbit/pkg/table/extension_darwin.go @@ -6,6 +6,7 @@ import ( "context" "github.com/fleetdm/fleet/v4/orbit/pkg/table/authdb" + "github.com/fleetdm/fleet/v4/orbit/pkg/table/codesign" "github.com/fleetdm/fleet/v4/orbit/pkg/table/csrutil_info" "github.com/fleetdm/fleet/v4/orbit/pkg/table/dataflattentable" "github.com/fleetdm/fleet/v4/orbit/pkg/table/diskutil/apfs" @@ -92,6 +93,8 @@ func PlatformTables(opts PluginOpts) ([]osquery.OsqueryPlugin, error) { // Table for parsing Apple Property List files, which are typically stored in ~/Library/Preferences/ dataflattentable.TablePlugin(log.Logger, dataflattentable.PlistType), // table name is "parse_plist" + + table.NewPlugin("codesign", codesign.Columns(), codesign.Generate), } // append platform specific tables diff --git a/schema/osquery_fleet_schema.json b/schema/osquery_fleet_schema.json index b538b1a00295..4624549bf730 100644 --- a/schema/osquery_fleet_schema.json +++ b/schema/osquery_fleet_schema.json @@ -4171,6 +4171,31 @@ "url": "https://fleetdm.com/tables/cis_audit", "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/cis_audit.yml" }, + { + "name": "codesign", + "platforms": [ + "darwin" + ], + "description": "Retrieves codesign information of a given .app path. It doesn't perform (expensive) verification, it just parses the signature from the 'Contents' folder using the \"codesign --display\" command.", + "columns": [ + { + "name": "path", + "type": "text", + "required": true, + "description": "Path is the absolute path to the app folder." + }, + { + "name": "team_identifier", + "type": "text", + "required": false, + "description": "Unique 10-character string generated by Apple that's assigned to a developer account to sign packages. This value is empty on unsigned applications and built-in Apple applications." + } + ], + "notes": "This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)).", + "evented": false, + "url": "https://fleetdm.com/tables/codesign", + "fleetRepoUrl": "https://github.com/fleetdm/fleet/blob/main/schema/tables/codesign.yml" + }, { "name": "connected_displays", "description": "Provides information about the connected displays of the machine.", diff --git a/schema/tables/codesign.yml b/schema/tables/codesign.yml new file mode 100644 index 000000000000..532da9bc4e8d --- /dev/null +++ b/schema/tables/codesign.yml @@ -0,0 +1,15 @@ +name: codesign +platforms: + - darwin +description: Retrieves codesign information of a given .app path. It doesn't perform (expensive) verification, it just parses the signature from the 'Contents' folder using the "codesign --display" command. +columns: + - name: path + type: text + required: true + description: Path is the absolute path to the app folder. + - name: team_identifier + type: text + required: false + description: Unique 10-character string generated by Apple that's assigned to a developer account to sign packages. This value is empty on unsigned applications and built-in Apple applications. +notes: This table is not a core osquery table. It is included as part of Fleet's agent ([fleetd](https://fleetdm.com/docs/get-started/anatomy#fleetd)). +evented: false diff --git a/server/datastore/mysql/migrations/tables/20241110152839_AddTeamIdentifierToHostSoftwareInstalledPaths.go b/server/datastore/mysql/migrations/tables/20241110152839_AddTeamIdentifierToHostSoftwareInstalledPaths.go new file mode 100644 index 000000000000..71fa8847e243 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20241110152839_AddTeamIdentifierToHostSoftwareInstalledPaths.go @@ -0,0 +1,23 @@ +package tables + +import ( + "database/sql" + "fmt" +) + +func init() { + MigrationClient.AddMigration(Up_20241110152839, Down_20241110152839) +} + +func Up_20241110152839(tx *sql.Tx) error { + if _, err := tx.Exec(` + ALTER TABLE host_software_installed_paths ADD COLUMN team_identifier VARCHAR(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT ''`, + ); err != nil { + return fmt.Errorf("failed to add team_identifier to host_software_installed_paths table: %w", err) + } + return nil +} + +func Down_20241110152839(tx *sql.Tx) error { + return nil +} diff --git a/server/datastore/mysql/schema.sql b/server/datastore/mysql/schema.sql index cbcc985cb204..7a4b13c3a727 100644 --- a/server/datastore/mysql/schema.sql +++ b/server/datastore/mysql/schema.sql @@ -587,9 +587,10 @@ CREATE TABLE `host_software_installed_paths` ( `host_id` int unsigned NOT NULL, `software_id` bigint unsigned NOT NULL, `installed_path` text COLLATE utf8mb4_unicode_ci NOT NULL, + `team_identifier` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', PRIMARY KEY (`id`), KEY `host_id_software_id_idx` (`host_id`,`software_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; @@ -1101,9 +1102,9 @@ CREATE TABLE `migration_status_tables` ( `tstamp` timestamp NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), UNIQUE KEY `id` (`id`) -) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=329 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +) /*!50100 TABLESPACE `innodb_system` */ ENGINE=InnoDB AUTO_INCREMENT=330 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; /*!40101 SET character_set_client = @saved_cs_client */; -INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241025141856,1,'2020-01-01 01:01:01'),(328,20241030102721,1,'2020-01-01 01:01:01'); +INSERT INTO `migration_status_tables` VALUES (1,0,1,'2020-01-01 01:01:01'),(2,20161118193812,1,'2020-01-01 01:01:01'),(3,20161118211713,1,'2020-01-01 01:01:01'),(4,20161118212436,1,'2020-01-01 01:01:01'),(5,20161118212515,1,'2020-01-01 01:01:01'),(6,20161118212528,1,'2020-01-01 01:01:01'),(7,20161118212538,1,'2020-01-01 01:01:01'),(8,20161118212549,1,'2020-01-01 01:01:01'),(9,20161118212557,1,'2020-01-01 01:01:01'),(10,20161118212604,1,'2020-01-01 01:01:01'),(11,20161118212613,1,'2020-01-01 01:01:01'),(12,20161118212621,1,'2020-01-01 01:01:01'),(13,20161118212630,1,'2020-01-01 01:01:01'),(14,20161118212641,1,'2020-01-01 01:01:01'),(15,20161118212649,1,'2020-01-01 01:01:01'),(16,20161118212656,1,'2020-01-01 01:01:01'),(17,20161118212758,1,'2020-01-01 01:01:01'),(18,20161128234849,1,'2020-01-01 01:01:01'),(19,20161230162221,1,'2020-01-01 01:01:01'),(20,20170104113816,1,'2020-01-01 01:01:01'),(21,20170105151732,1,'2020-01-01 01:01:01'),(22,20170108191242,1,'2020-01-01 01:01:01'),(23,20170109094020,1,'2020-01-01 01:01:01'),(24,20170109130438,1,'2020-01-01 01:01:01'),(25,20170110202752,1,'2020-01-01 01:01:01'),(26,20170111133013,1,'2020-01-01 01:01:01'),(27,20170117025759,1,'2020-01-01 01:01:01'),(28,20170118191001,1,'2020-01-01 01:01:01'),(29,20170119234632,1,'2020-01-01 01:01:01'),(30,20170124230432,1,'2020-01-01 01:01:01'),(31,20170127014618,1,'2020-01-01 01:01:01'),(32,20170131232841,1,'2020-01-01 01:01:01'),(33,20170223094154,1,'2020-01-01 01:01:01'),(34,20170306075207,1,'2020-01-01 01:01:01'),(35,20170309100733,1,'2020-01-01 01:01:01'),(36,20170331111922,1,'2020-01-01 01:01:01'),(37,20170502143928,1,'2020-01-01 01:01:01'),(38,20170504130602,1,'2020-01-01 01:01:01'),(39,20170509132100,1,'2020-01-01 01:01:01'),(40,20170519105647,1,'2020-01-01 01:01:01'),(41,20170519105648,1,'2020-01-01 01:01:01'),(42,20170831234300,1,'2020-01-01 01:01:01'),(43,20170831234301,1,'2020-01-01 01:01:01'),(44,20170831234303,1,'2020-01-01 01:01:01'),(45,20171116163618,1,'2020-01-01 01:01:01'),(46,20171219164727,1,'2020-01-01 01:01:01'),(47,20180620164811,1,'2020-01-01 01:01:01'),(48,20180620175054,1,'2020-01-01 01:01:01'),(49,20180620175055,1,'2020-01-01 01:01:01'),(50,20191010101639,1,'2020-01-01 01:01:01'),(51,20191010155147,1,'2020-01-01 01:01:01'),(52,20191220130734,1,'2020-01-01 01:01:01'),(53,20200311140000,1,'2020-01-01 01:01:01'),(54,20200405120000,1,'2020-01-01 01:01:01'),(55,20200407120000,1,'2020-01-01 01:01:01'),(56,20200420120000,1,'2020-01-01 01:01:01'),(57,20200504120000,1,'2020-01-01 01:01:01'),(58,20200512120000,1,'2020-01-01 01:01:01'),(59,20200707120000,1,'2020-01-01 01:01:01'),(60,20201011162341,1,'2020-01-01 01:01:01'),(61,20201021104586,1,'2020-01-01 01:01:01'),(62,20201102112520,1,'2020-01-01 01:01:01'),(63,20201208121729,1,'2020-01-01 01:01:01'),(64,20201215091637,1,'2020-01-01 01:01:01'),(65,20210119174155,1,'2020-01-01 01:01:01'),(66,20210326182902,1,'2020-01-01 01:01:01'),(67,20210421112652,1,'2020-01-01 01:01:01'),(68,20210506095025,1,'2020-01-01 01:01:01'),(69,20210513115729,1,'2020-01-01 01:01:01'),(70,20210526113559,1,'2020-01-01 01:01:01'),(71,20210601000001,1,'2020-01-01 01:01:01'),(72,20210601000002,1,'2020-01-01 01:01:01'),(73,20210601000003,1,'2020-01-01 01:01:01'),(74,20210601000004,1,'2020-01-01 01:01:01'),(75,20210601000005,1,'2020-01-01 01:01:01'),(76,20210601000006,1,'2020-01-01 01:01:01'),(77,20210601000007,1,'2020-01-01 01:01:01'),(78,20210601000008,1,'2020-01-01 01:01:01'),(79,20210606151329,1,'2020-01-01 01:01:01'),(80,20210616163757,1,'2020-01-01 01:01:01'),(81,20210617174723,1,'2020-01-01 01:01:01'),(82,20210622160235,1,'2020-01-01 01:01:01'),(83,20210623100031,1,'2020-01-01 01:01:01'),(84,20210623133615,1,'2020-01-01 01:01:01'),(85,20210708143152,1,'2020-01-01 01:01:01'),(86,20210709124443,1,'2020-01-01 01:01:01'),(87,20210712155608,1,'2020-01-01 01:01:01'),(88,20210714102108,1,'2020-01-01 01:01:01'),(89,20210719153709,1,'2020-01-01 01:01:01'),(90,20210721171531,1,'2020-01-01 01:01:01'),(91,20210723135713,1,'2020-01-01 01:01:01'),(92,20210802135933,1,'2020-01-01 01:01:01'),(93,20210806112844,1,'2020-01-01 01:01:01'),(94,20210810095603,1,'2020-01-01 01:01:01'),(95,20210811150223,1,'2020-01-01 01:01:01'),(96,20210818151827,1,'2020-01-01 01:01:01'),(97,20210818151828,1,'2020-01-01 01:01:01'),(98,20210818182258,1,'2020-01-01 01:01:01'),(99,20210819131107,1,'2020-01-01 01:01:01'),(100,20210819143446,1,'2020-01-01 01:01:01'),(101,20210903132338,1,'2020-01-01 01:01:01'),(102,20210915144307,1,'2020-01-01 01:01:01'),(103,20210920155130,1,'2020-01-01 01:01:01'),(104,20210927143115,1,'2020-01-01 01:01:01'),(105,20210927143116,1,'2020-01-01 01:01:01'),(106,20211013133706,1,'2020-01-01 01:01:01'),(107,20211013133707,1,'2020-01-01 01:01:01'),(108,20211102135149,1,'2020-01-01 01:01:01'),(109,20211109121546,1,'2020-01-01 01:01:01'),(110,20211110163320,1,'2020-01-01 01:01:01'),(111,20211116184029,1,'2020-01-01 01:01:01'),(112,20211116184030,1,'2020-01-01 01:01:01'),(113,20211202092042,1,'2020-01-01 01:01:01'),(114,20211202181033,1,'2020-01-01 01:01:01'),(115,20211207161856,1,'2020-01-01 01:01:01'),(116,20211216131203,1,'2020-01-01 01:01:01'),(117,20211221110132,1,'2020-01-01 01:01:01'),(118,20220107155700,1,'2020-01-01 01:01:01'),(119,20220125105650,1,'2020-01-01 01:01:01'),(120,20220201084510,1,'2020-01-01 01:01:01'),(121,20220208144830,1,'2020-01-01 01:01:01'),(122,20220208144831,1,'2020-01-01 01:01:01'),(123,20220215152203,1,'2020-01-01 01:01:01'),(124,20220223113157,1,'2020-01-01 01:01:01'),(125,20220307104655,1,'2020-01-01 01:01:01'),(126,20220309133956,1,'2020-01-01 01:01:01'),(127,20220316155700,1,'2020-01-01 01:01:01'),(128,20220323152301,1,'2020-01-01 01:01:01'),(129,20220330100659,1,'2020-01-01 01:01:01'),(130,20220404091216,1,'2020-01-01 01:01:01'),(131,20220419140750,1,'2020-01-01 01:01:01'),(132,20220428140039,1,'2020-01-01 01:01:01'),(133,20220503134048,1,'2020-01-01 01:01:01'),(134,20220524102918,1,'2020-01-01 01:01:01'),(135,20220526123327,1,'2020-01-01 01:01:01'),(136,20220526123328,1,'2020-01-01 01:01:01'),(137,20220526123329,1,'2020-01-01 01:01:01'),(138,20220608113128,1,'2020-01-01 01:01:01'),(139,20220627104817,1,'2020-01-01 01:01:01'),(140,20220704101843,1,'2020-01-01 01:01:01'),(141,20220708095046,1,'2020-01-01 01:01:01'),(142,20220713091130,1,'2020-01-01 01:01:01'),(143,20220802135510,1,'2020-01-01 01:01:01'),(144,20220818101352,1,'2020-01-01 01:01:01'),(145,20220822161445,1,'2020-01-01 01:01:01'),(146,20220831100036,1,'2020-01-01 01:01:01'),(147,20220831100151,1,'2020-01-01 01:01:01'),(148,20220908181826,1,'2020-01-01 01:01:01'),(149,20220914154915,1,'2020-01-01 01:01:01'),(150,20220915165115,1,'2020-01-01 01:01:01'),(151,20220915165116,1,'2020-01-01 01:01:01'),(152,20220928100158,1,'2020-01-01 01:01:01'),(153,20221014084130,1,'2020-01-01 01:01:01'),(154,20221027085019,1,'2020-01-01 01:01:01'),(155,20221101103952,1,'2020-01-01 01:01:01'),(156,20221104144401,1,'2020-01-01 01:01:01'),(157,20221109100749,1,'2020-01-01 01:01:01'),(158,20221115104546,1,'2020-01-01 01:01:01'),(159,20221130114928,1,'2020-01-01 01:01:01'),(160,20221205112142,1,'2020-01-01 01:01:01'),(161,20221216115820,1,'2020-01-01 01:01:01'),(162,20221220195934,1,'2020-01-01 01:01:01'),(163,20221220195935,1,'2020-01-01 01:01:01'),(164,20221223174807,1,'2020-01-01 01:01:01'),(165,20221227163855,1,'2020-01-01 01:01:01'),(166,20221227163856,1,'2020-01-01 01:01:01'),(167,20230202224725,1,'2020-01-01 01:01:01'),(168,20230206163608,1,'2020-01-01 01:01:01'),(169,20230214131519,1,'2020-01-01 01:01:01'),(170,20230303135738,1,'2020-01-01 01:01:01'),(171,20230313135301,1,'2020-01-01 01:01:01'),(172,20230313141819,1,'2020-01-01 01:01:01'),(173,20230315104937,1,'2020-01-01 01:01:01'),(174,20230317173844,1,'2020-01-01 01:01:01'),(175,20230320133602,1,'2020-01-01 01:01:01'),(176,20230330100011,1,'2020-01-01 01:01:01'),(177,20230330134823,1,'2020-01-01 01:01:01'),(178,20230405232025,1,'2020-01-01 01:01:01'),(179,20230408084104,1,'2020-01-01 01:01:01'),(180,20230411102858,1,'2020-01-01 01:01:01'),(181,20230421155932,1,'2020-01-01 01:01:01'),(182,20230425082126,1,'2020-01-01 01:01:01'),(183,20230425105727,1,'2020-01-01 01:01:01'),(184,20230501154913,1,'2020-01-01 01:01:01'),(185,20230503101418,1,'2020-01-01 01:01:01'),(186,20230515144206,1,'2020-01-01 01:01:01'),(187,20230517140952,1,'2020-01-01 01:01:01'),(188,20230517152807,1,'2020-01-01 01:01:01'),(189,20230518114155,1,'2020-01-01 01:01:01'),(190,20230520153236,1,'2020-01-01 01:01:01'),(191,20230525151159,1,'2020-01-01 01:01:01'),(192,20230530122103,1,'2020-01-01 01:01:01'),(193,20230602111827,1,'2020-01-01 01:01:01'),(194,20230608103123,1,'2020-01-01 01:01:01'),(195,20230629140529,1,'2020-01-01 01:01:01'),(196,20230629140530,1,'2020-01-01 01:01:01'),(197,20230711144622,1,'2020-01-01 01:01:01'),(198,20230721135421,1,'2020-01-01 01:01:01'),(199,20230721161508,1,'2020-01-01 01:01:01'),(200,20230726115701,1,'2020-01-01 01:01:01'),(201,20230807100822,1,'2020-01-01 01:01:01'),(202,20230814150442,1,'2020-01-01 01:01:01'),(203,20230823122728,1,'2020-01-01 01:01:01'),(204,20230906152143,1,'2020-01-01 01:01:01'),(205,20230911163618,1,'2020-01-01 01:01:01'),(206,20230912101759,1,'2020-01-01 01:01:01'),(207,20230915101341,1,'2020-01-01 01:01:01'),(208,20230918132351,1,'2020-01-01 01:01:01'),(209,20231004144339,1,'2020-01-01 01:01:01'),(210,20231009094541,1,'2020-01-01 01:01:01'),(211,20231009094542,1,'2020-01-01 01:01:01'),(212,20231009094543,1,'2020-01-01 01:01:01'),(213,20231009094544,1,'2020-01-01 01:01:01'),(214,20231016091915,1,'2020-01-01 01:01:01'),(215,20231024174135,1,'2020-01-01 01:01:01'),(216,20231025120016,1,'2020-01-01 01:01:01'),(217,20231025160156,1,'2020-01-01 01:01:01'),(218,20231031165350,1,'2020-01-01 01:01:01'),(219,20231106144110,1,'2020-01-01 01:01:01'),(220,20231107130934,1,'2020-01-01 01:01:01'),(221,20231109115838,1,'2020-01-01 01:01:01'),(222,20231121054530,1,'2020-01-01 01:01:01'),(223,20231122101320,1,'2020-01-01 01:01:01'),(224,20231130132828,1,'2020-01-01 01:01:01'),(225,20231130132931,1,'2020-01-01 01:01:01'),(226,20231204155427,1,'2020-01-01 01:01:01'),(227,20231206142340,1,'2020-01-01 01:01:01'),(228,20231207102320,1,'2020-01-01 01:01:01'),(229,20231207102321,1,'2020-01-01 01:01:01'),(230,20231207133731,1,'2020-01-01 01:01:01'),(231,20231212094238,1,'2020-01-01 01:01:01'),(232,20231212095734,1,'2020-01-01 01:01:01'),(233,20231212161121,1,'2020-01-01 01:01:01'),(234,20231215122713,1,'2020-01-01 01:01:01'),(235,20231219143041,1,'2020-01-01 01:01:01'),(236,20231224070653,1,'2020-01-01 01:01:01'),(237,20240110134315,1,'2020-01-01 01:01:01'),(238,20240119091637,1,'2020-01-01 01:01:01'),(239,20240126020642,1,'2020-01-01 01:01:01'),(240,20240126020643,1,'2020-01-01 01:01:01'),(241,20240129162819,1,'2020-01-01 01:01:01'),(242,20240130115133,1,'2020-01-01 01:01:01'),(243,20240131083822,1,'2020-01-01 01:01:01'),(244,20240205095928,1,'2020-01-01 01:01:01'),(245,20240205121956,1,'2020-01-01 01:01:01'),(246,20240209110212,1,'2020-01-01 01:01:01'),(247,20240212111533,1,'2020-01-01 01:01:01'),(248,20240221112844,1,'2020-01-01 01:01:01'),(249,20240222073518,1,'2020-01-01 01:01:01'),(250,20240222135115,1,'2020-01-01 01:01:01'),(251,20240226082255,1,'2020-01-01 01:01:01'),(252,20240228082706,1,'2020-01-01 01:01:01'),(253,20240301173035,1,'2020-01-01 01:01:01'),(254,20240302111134,1,'2020-01-01 01:01:01'),(255,20240312103753,1,'2020-01-01 01:01:01'),(256,20240313143416,1,'2020-01-01 01:01:01'),(257,20240314085226,1,'2020-01-01 01:01:01'),(258,20240314151747,1,'2020-01-01 01:01:01'),(259,20240320145650,1,'2020-01-01 01:01:01'),(260,20240327115530,1,'2020-01-01 01:01:01'),(261,20240327115617,1,'2020-01-01 01:01:01'),(262,20240408085837,1,'2020-01-01 01:01:01'),(263,20240415104633,1,'2020-01-01 01:01:01'),(264,20240430111727,1,'2020-01-01 01:01:01'),(265,20240515200020,1,'2020-01-01 01:01:01'),(266,20240521143023,1,'2020-01-01 01:01:01'),(267,20240521143024,1,'2020-01-01 01:01:01'),(268,20240601174138,1,'2020-01-01 01:01:01'),(269,20240607133721,1,'2020-01-01 01:01:01'),(270,20240612150059,1,'2020-01-01 01:01:01'),(271,20240613162201,1,'2020-01-01 01:01:01'),(272,20240613172616,1,'2020-01-01 01:01:01'),(273,20240618142419,1,'2020-01-01 01:01:01'),(274,20240625093543,1,'2020-01-01 01:01:01'),(275,20240626195531,1,'2020-01-01 01:01:01'),(276,20240702123921,1,'2020-01-01 01:01:01'),(277,20240703154849,1,'2020-01-01 01:01:01'),(278,20240707134035,1,'2020-01-01 01:01:01'),(279,20240707134036,1,'2020-01-01 01:01:01'),(280,20240709124958,1,'2020-01-01 01:01:01'),(281,20240709132642,1,'2020-01-01 01:01:01'),(282,20240709183940,1,'2020-01-01 01:01:01'),(283,20240710155623,1,'2020-01-01 01:01:01'),(284,20240723102712,1,'2020-01-01 01:01:01'),(285,20240725152735,1,'2020-01-01 01:01:01'),(286,20240725182118,1,'2020-01-01 01:01:01'),(287,20240726100517,1,'2020-01-01 01:01:01'),(288,20240730171504,1,'2020-01-01 01:01:01'),(289,20240730174056,1,'2020-01-01 01:01:01'),(290,20240730215453,1,'2020-01-01 01:01:01'),(291,20240730374423,1,'2020-01-01 01:01:01'),(292,20240801115359,1,'2020-01-01 01:01:01'),(293,20240802101043,1,'2020-01-01 01:01:01'),(294,20240802113716,1,'2020-01-01 01:01:01'),(295,20240814135330,1,'2020-01-01 01:01:01'),(296,20240815000000,1,'2020-01-01 01:01:01'),(297,20240815000001,1,'2020-01-01 01:01:01'),(298,20240816103247,1,'2020-01-01 01:01:01'),(299,20240820091218,1,'2020-01-01 01:01:01'),(300,20240826111228,1,'2020-01-01 01:01:01'),(301,20240826160025,1,'2020-01-01 01:01:01'),(302,20240829165448,1,'2020-01-01 01:01:01'),(303,20240829165605,1,'2020-01-01 01:01:01'),(304,20240829165715,1,'2020-01-01 01:01:01'),(305,20240829165930,1,'2020-01-01 01:01:01'),(306,20240829170023,1,'2020-01-01 01:01:01'),(307,20240829170033,1,'2020-01-01 01:01:01'),(308,20240829170044,1,'2020-01-01 01:01:01'),(309,20240905105135,1,'2020-01-01 01:01:01'),(310,20240905140514,1,'2020-01-01 01:01:01'),(311,20240905200000,1,'2020-01-01 01:01:01'),(312,20240905200001,1,'2020-01-01 01:01:01'),(313,20241002104104,1,'2020-01-01 01:01:01'),(314,20241002104105,1,'2020-01-01 01:01:01'),(315,20241002104106,1,'2020-01-01 01:01:01'),(316,20241002210000,1,'2020-01-01 01:01:01'),(317,20241003145349,1,'2020-01-01 01:01:01'),(318,20241004005000,1,'2020-01-01 01:01:01'),(319,20241008083925,1,'2020-01-01 01:01:01'),(320,20241009090010,1,'2020-01-01 01:01:01'),(321,20241017163402,1,'2020-01-01 01:01:01'),(322,20241021224359,1,'2020-01-01 01:01:01'),(323,20241022140321,1,'2020-01-01 01:01:01'),(324,20241025111236,1,'2020-01-01 01:01:01'),(325,20241025112748,1,'2020-01-01 01:01:01'),(326,20241025141855,1,'2020-01-01 01:01:01'),(327,20241025141856,1,'2020-01-01 01:01:01'),(328,20241030102721,1,'2020-01-01 01:01:01'),(329,20241110152839,1,'2020-01-01 01:01:01'); /*!40101 SET @saved_cs_client = @@character_set_client */; /*!50503 SET character_set_client = utf8mb4 */; CREATE TABLE `mobile_device_management_solutions` ( diff --git a/server/datastore/mysql/software.go b/server/datastore/mysql/software.go index d6cd45f962f8..d89bf398cf37 100644 --- a/server/datastore/mysql/software.go +++ b/server/datastore/mysql/software.go @@ -91,7 +91,7 @@ func (ds *Datastore) getHostSoftwareInstalledPaths( error, ) { stmt := ` - SELECT t.id, t.host_id, t.software_id, t.installed_path + SELECT t.id, t.host_id, t.software_id, t.installed_path, t.team_identifier FROM host_software_installed_paths t WHERE t.host_id = ? ` @@ -145,7 +145,10 @@ func hostSoftwareInstalledPathsDelta( continue } - key := fmt.Sprintf("%s%s%s", r.InstalledPath, fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + key := fmt.Sprintf( + "%s%s%s%s%s", + r.InstalledPath, fleet.SoftwareFieldSeparator, r.TeamIdentifier, fleet.SoftwareFieldSeparator, s.ToUniqueStr(), + ) iSPathLookup[key] = r // Anything stored but not reported should be deleted @@ -155,8 +158,8 @@ func hostSoftwareInstalledPathsDelta( } for key := range reported { - parts := strings.SplitN(key, fleet.SoftwareFieldSeparator, 2) - iSPath, unqStr := parts[0], parts[1] + parts := strings.SplitN(key, fleet.SoftwareFieldSeparator, 3) + installedPath, teamIdentifier, unqStr := parts[0], parts[1], parts[2] // Shouldn't be possible ... everything 'reported' should be in the the software table // because this executes after 'ds.UpdateHostSoftware' @@ -172,9 +175,10 @@ func hostSoftwareInstalledPathsDelta( } toInsert = append(toInsert, fleet.HostSoftwareInstalledPath{ - HostID: hostID, - SoftwareID: s.ID, - InstalledPath: iSPath, + HostID: hostID, + SoftwareID: s.ID, + InstalledPath: installedPath, + TeamIdentifier: teamIdentifier, }) } @@ -211,7 +215,7 @@ func insertHostSoftwareInstalledPaths( return nil } - stmt := "INSERT INTO host_software_installed_paths (host_id, software_id, installed_path) VALUES %s" + stmt := "INSERT INTO host_software_installed_paths (host_id, software_id, installed_path, team_identifier) VALUES %s" batchSize := 500 for i := 0; i < len(toInsert); i += batchSize { @@ -223,10 +227,10 @@ func insertHostSoftwareInstalledPaths( var args []interface{} for _, v := range batch { - args = append(args, v.HostID, v.SoftwareID, v.InstalledPath) + args = append(args, v.HostID, v.SoftwareID, v.InstalledPath, v.TeamIdentifier) } - placeHolders := strings.TrimSuffix(strings.Repeat("(?, ?, ?), ", len(batch)), ", ") + placeHolders := strings.TrimSuffix(strings.Repeat("(?, ?, ?, ?), ", len(batch)), ", ") stmt := fmt.Sprintf(stmt, placeHolders) _, err := tx.ExecContext(ctx, stmt, args...) @@ -639,7 +643,19 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB( ) // INSERT IGNORE is used to avoid duplicate key errors, which may occur since our previous read came from the replica. stmt := fmt.Sprintf( - "INSERT IGNORE INTO software (name, version, source, `release`, vendor, arch, bundle_identifier, extension_id, browser, title_id, checksum) VALUES %s", + `INSERT IGNORE INTO software ( + name, + version, + source, + `+"`release`"+`, + vendor, + arch, + bundle_identifier, + extension_id, + browser, + title_id, + checksum + ) VALUES %s`, values, ) args := make([]interface{}, 0, totalToProcess*numberOfArgsPerSoftware) @@ -1228,16 +1244,22 @@ func (ds *Datastore) LoadHostSoftware(ctx context.Context, host *fleet.Host, inc return err } - lookup := make(map[uint][]string) + installedPathsList := make(map[uint][]string) + pathSignatureInformation := make(map[uint][]fleet.PathSignatureInformation) for _, ip := range installedPaths { - lookup[ip.SoftwareID] = append(lookup[ip.SoftwareID], ip.InstalledPath) + installedPathsList[ip.SoftwareID] = append(installedPathsList[ip.SoftwareID], ip.InstalledPath) + pathSignatureInformation[ip.SoftwareID] = append(pathSignatureInformation[ip.SoftwareID], fleet.PathSignatureInformation{ + InstalledPath: ip.InstalledPath, + TeamIdentifier: ip.TeamIdentifier, + }) } host.Software = make([]fleet.HostSoftwareEntry, 0, len(software)) for _, s := range software { host.Software = append(host.Software, fleet.HostSoftwareEntry{ - Software: s, - InstalledPaths: lookup[s.ID], + Software: s, + InstalledPaths: installedPathsList[s.ID], + PathSignatureInformation: pathSignatureInformation[s.ID], }) } return nil @@ -1283,7 +1305,7 @@ func (ds *Datastore) AllSoftwareIterator( var args []interface{} stmt := `SELECT - s.id, s.name, s.version, s.source, s.bundle_identifier, s.release, s.arch, s.vendor, s.browser, s.extension_id, s.title_id , + s.id, s.name, s.version, s.source, s.bundle_identifier, s.release, s.arch, s.vendor, s.browser, s.extension_id, s.title_id, COALESCE(sc.cpe, '') AS generated_cpe FROM software s LEFT JOIN software_cpe sc ON (s.id=sc.software_id)` @@ -2524,6 +2546,8 @@ INNER JOIN software_cve scve ON scve.software_id = s.id st.id as software_title_id, s.id as software_id, s.version, + s.bundle_identifier, + s.source, hs.last_opened_at FROM software s @@ -2588,7 +2612,8 @@ INNER JOIN software_cve scve ON scve.software_id = s.id const pathsStmt = ` SELECT hsip.software_id, - hsip.installed_path + hsip.installed_path, + hsip.team_identifier FROM host_software_installed_paths hsip WHERE @@ -2598,8 +2623,9 @@ INNER JOIN software_cve scve ON scve.software_id = s.id software_id, installed_path ` type installedPath struct { - SoftwareID uint `db:"software_id"` - InstalledPath string `db:"installed_path"` + SoftwareID uint `db:"software_id"` + InstalledPath string `db:"installed_path"` + TeamIdentifier string `db:"team_identifier"` } var installedPaths []installedPath stmt, args, err = sqlx.In(pathsStmt, host.ID, softwareIDs) @@ -2614,6 +2640,12 @@ INNER JOIN software_cve scve ON scve.software_id = s.id for _, path := range installedPaths { ver := bySoftwareID[path.SoftwareID] ver.InstalledPaths = append(ver.InstalledPaths, path.InstalledPath) + if ver.Source == "apps" { + ver.SignatureInformation = append(ver.SignatureInformation, fleet.PathSignatureInformation{ + InstalledPath: path.InstalledPath, + TeamIdentifier: path.TeamIdentifier, + }) + } } } } diff --git a/server/datastore/mysql/software_test.go b/server/datastore/mysql/software_test.go index 4e99fe4b6a3b..a613ba82cbe4 100644 --- a/server/datastore/mysql/software_test.go +++ b/server/datastore/mysql/software_test.go @@ -58,10 +58,10 @@ func TestSoftware(t *testing.T) { {"DeleteSoftwareCPEs", testDeleteSoftwareCPEs}, {"SoftwareByIDNoDuplicatedVulns", testSoftwareByIDNoDuplicatedVulns}, {"SoftwareByIDIncludesCVEPublishedDate", testSoftwareByIDIncludesCVEPublishedDate}, - {"getHostSoftwareInstalledPaths", testGetHostSoftwareInstalledPaths}, - {"hostSoftwareInstalledPathsDelta", testHostSoftwareInstalledPathsDelta}, - {"deleteHostSoftwareInstalledPaths", testDeleteHostSoftwareInstalledPaths}, - {"insertHostSoftwareInstalledPaths", testInsertHostSoftwareInstalledPaths}, + {"GetHostSoftwareInstalledPaths", testGetHostSoftwareInstalledPaths}, + {"HostSoftwareInstalledPathsDelta", testHostSoftwareInstalledPathsDelta}, + {"DeleteHostSoftwareInstalledPaths", testDeleteHostSoftwareInstalledPaths}, + {"InsertHostSoftwareInstalledPaths", testInsertHostSoftwareInstalledPaths}, {"VerifySoftwareChecksum", testVerifySoftwareChecksum}, {"ListHostSoftware", testListHostSoftware}, {"ListIOSHostSoftware", testListIOSHostSoftware}, @@ -1342,7 +1342,7 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { // Insert paths for software1 s1Paths := map[string]struct{}{} for _, s := range software1 { - key := fmt.Sprintf("%s%s%s", fmt.Sprintf("/some/path/%s", s.Name), fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + key := fmt.Sprintf("%s%s%s%s%s", fmt.Sprintf("/some/path/%s", s.Name), fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, s.ToUniqueStr()) s1Paths[key] = struct{}{} } require.NoError(t, ds.UpdateHostSoftwareInstalledPaths(context.Background(), host1.ID, s1Paths, mutationResults)) @@ -1353,7 +1353,7 @@ func insertVulnSoftwareForTest(t *testing.T, ds *Datastore) { // Insert paths for software2 s2Paths := map[string]struct{}{} for _, s := range software2 { - key := fmt.Sprintf("%s%s%s", fmt.Sprintf("/some/path/%s", s.Name), fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + key := fmt.Sprintf("%s%s%s%s%s", fmt.Sprintf("/some/path/%s", s.Name), fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, s.ToUniqueStr()) s2Paths[key] = struct{}{} } require.NoError(t, ds.UpdateHostSoftwareInstalledPaths(context.Background(), host2.ID, s2Paths, mutationResults)) @@ -2733,9 +2733,9 @@ func testHostSoftwareInstalledPathsDelta(t *testing.T, ds *Datastore) { t.Run("host has no software but some paths were reported", func(t *testing.T) { reported := make(map[string]struct{}) - reported[fmt.Sprintf("/some/path/%d%s%s", software[0].ID, fleet.SoftwareFieldSeparator, software[0].ToUniqueStr())] = struct{}{} - reported[fmt.Sprintf("/some/path/%d%s%s", software[1].ID+1, fleet.SoftwareFieldSeparator, software[1].ToUniqueStr())] = struct{}{} - reported[fmt.Sprintf("/some/path/%d%s%s", software[2].ID, fleet.SoftwareFieldSeparator, software[2].ToUniqueStr())] = struct{}{} + reported[fmt.Sprintf("/some/path/%d%s%s%s%s", software[0].ID, fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, software[0].ToUniqueStr())] = struct{}{} + reported[fmt.Sprintf("/some/path/%d%s%s%s%s", software[1].ID+1, fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, software[1].ToUniqueStr())] = struct{}{} + reported[fmt.Sprintf("/some/path/%d%s%s%s%s", software[2].ID, fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, software[2].ToUniqueStr())] = struct{}{} var stored []fleet.HostSoftwareInstalledPath _, _, err := hostSoftwareInstalledPathsDelta(host.ID, reported, stored, nil) @@ -2744,7 +2744,7 @@ func testHostSoftwareInstalledPathsDelta(t *testing.T, ds *Datastore) { t.Run("we have some deltas", func(t *testing.T) { getKey := func(s fleet.Software, change uint) string { - return fmt.Sprintf("/some/path/%d%s%s", s.ID+change, fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + return fmt.Sprintf("/some/path/%d%s%s%s%s", s.ID+change, fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, s.ToUniqueStr()) } reported := make(map[string]struct{}) reported[getKey(software[0], 0)] = struct{}{} @@ -3308,7 +3308,7 @@ func testListHostSoftware(t *testing.T, ds *Datastore) { installPaths := make([]string, 0, len(software)) for _, s := range software { path := fmt.Sprintf("/some/path/%s", s.Name) - key := fmt.Sprintf("%s%s%s", path, fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + key := fmt.Sprintf("%s%s%s%s%s", path, fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, s.ToUniqueStr()) swPaths[key] = struct{}{} installPaths = append(installPaths, path) } diff --git a/server/fleet/datastore.go b/server/fleet/datastore.go index c962d4119c09..2cfaf6deb3ff 100644 --- a/server/fleet/datastore.go +++ b/server/fleet/datastore.go @@ -842,9 +842,12 @@ type Datastore interface { // UpdateHostSoftwareInstalledPaths looks at all software for 'hostID' and based on the contents of // 'reported', either inserts or deletes the corresponding entries in the // 'host_software_installed_paths' table. 'reported' is a set of - // 'software.ToUniqueStr()--installed_path' strings. 'mutationResults' contains the software inventory of + // 'installed_path\0team_identifier\0software.ToUniqueStr()' strings. 'mutationResults' contains the software inventory of // the host (pre-mutations) and the mutations performed after calling 'UpdateHostSoftware', // it is used as DB optimization. + // + // TODO(lucas): We should amend UpdateHostSoftwareInstalledPaths to just accept raw information + // otherwise the caller has to assemble the reported set the same way in all places where it's used. UpdateHostSoftwareInstalledPaths(ctx context.Context, hostID uint, reported map[string]struct{}, mutationResults *UpdateHostSoftwareDBResult) error // UpdateHost updates a host. diff --git a/server/fleet/hosts.go b/server/fleet/hosts.go index 3ff287c9f10d..e402af547218 100644 --- a/server/fleet/hosts.go +++ b/server/fleet/hosts.go @@ -1185,6 +1185,9 @@ type HostSoftwareInstalledPath struct { SoftwareID uint `db:"software_id"` // InstalledPath is the file system path where the software is installed InstalledPath string `db:"installed_path"` + // TeamIdentifier (not to be confused with Fleet's team IDs) is the Apple's "Team ID" (aka "Developer ID" + // or "Signing ID") of signed applications, see https://developer.apple.com/help/account/manage-your-team/locate-your-team-id. + TeamIdentifier string `db:"team_identifier"` } // HostMacOSProfile represents a macOS profile installed on a host as reported by the macos_profiles diff --git a/server/fleet/software.go b/server/fleet/software.go index 487c1a50e9f6..04b40771312c 100644 --- a/server/fleet/software.go +++ b/server/fleet/software.go @@ -28,6 +28,10 @@ const ( SoftwareReleaseMaxLength = 64 SoftwareVendorMaxLength = 114 SoftwareArchMaxLength = 16 + + // SoftwareTeamIdentifierMaxLength is the max length for Apple's Team ID, + // see https://developer.apple.com/help/account/manage-your-team/locate-your-team-id + SoftwareTeamIdentifierMaxLength = 10 ) type Vulnerabilities []CVE @@ -271,7 +275,13 @@ type HostSoftwareEntry struct { Software // Where this software was installed on the host, value is derived from the // host_software_installed_paths table. - InstalledPaths []string `json:"installed_paths"` + InstalledPaths []string `json:"installed_paths"` + PathSignatureInformation []PathSignatureInformation `json:"signature_information"` +} + +type PathSignatureInformation struct { + InstalledPath string `json:"installed_path"` + TeamIdentifier string `json:"team_identifier"` } // HostSoftware is the set of software installed on a specific host @@ -383,9 +393,10 @@ func ParseSoftwareLastOpenedAtRowValue(value string) (time.Time, error) { // // All fields are trimmed to fit on Fleet's database. // The vendor field is currently trimmed by removing the extra characters and adding `...` at the end. -func SoftwareFromOsqueryRow(name, version, source, vendor, installedPath, release, arch, bundleIdentifier, extensionId, browser, lastOpenedAt string) ( - *Software, error, -) { +func SoftwareFromOsqueryRow( + name, version, source, vendor, installedPath, release, arch, + bundleIdentifier, extensionId, browser, lastOpenedAt string, +) (*Software, error) { if name == "" { return nil, errors.New("host reported software with empty name") } diff --git a/server/fleet/software_installer.go b/server/fleet/software_installer.go index aae130902a1c..835762c1549e 100644 --- a/server/fleet/software_installer.go +++ b/server/fleet/software_installer.go @@ -480,15 +480,18 @@ type HostSoftwareUninstall struct { UninstalledAt time.Time `json:"uninstalled_at"` } -// HostSoftwareInstalledVersion represents a version of software installed on a -// host. +// HostSoftwareInstalledVersion represents a version of software installed on a host. type HostSoftwareInstalledVersion struct { - SoftwareID uint `json:"-" db:"software_id"` - SoftwareTitleID uint `json:"-" db:"software_title_id"` - Version string `json:"version" db:"version"` - LastOpenedAt *time.Time `json:"last_opened_at" db:"last_opened_at"` - Vulnerabilities []string `json:"vulnerabilities" db:"vulnerabilities"` - InstalledPaths []string `json:"installed_paths" db:"installed_paths"` + SoftwareID uint `json:"-" db:"software_id"` + SoftwareTitleID uint `json:"-" db:"software_title_id"` + Source string `json:"-" db:"source"` + Version string `json:"version" db:"version"` + BundleIdentifier string `json:"bundle_identifier,omitempty" db:"bundle_identifier"` + LastOpenedAt *time.Time `json:"last_opened_at" db:"last_opened_at"` + + Vulnerabilities []string `json:"vulnerabilities" db:"vulnerabilities"` + InstalledPaths []string `json:"installed_paths"` + SignatureInformation []PathSignatureInformation `json:"signature_information,omitempty"` } // HostSoftwareInstallResultPayload is the payload provided by fleetd to record diff --git a/server/service/integration_core_test.go b/server/service/integration_core_test.go index 2e08b4ef9dc3..45c56a3d7642 100644 --- a/server/service/integration_core_test.go +++ b/server/service/integration_core_test.go @@ -12292,3 +12292,108 @@ func (s *integrationTestSuite) TestHostWithNoPoliciesClearsPolicyCounts() { require.Len(t, listHostsResp.Hosts, 1) require.Equal(t, uint64(0), listHostsResp.Hosts[0].FailingPoliciesCount) } + +func (s *integrationTestSuite) TestHostSoftwareWithTeamIdentifier() { + t := s.T() + ctx := context.Background() + + host, err := s.ds.NewHost(ctx, &fleet.Host{ + NodeKey: ptr.String(t.Name()), + OsqueryHostID: ptr.String(t.Name()), + UUID: t.Name(), + Hostname: t.Name() + "foo.local", + Platform: "darwin", + }) + require.NoError(t, err) + + safariApp := fleet.Software{ + Name: "Safari.app", + BundleIdentifier: "com.apple.safari", + Version: "18.1", + Source: "apps", + } + googleChromeApp := fleet.Software{ + Name: "Google Chrome.app", + BundleIdentifier: "com.google.Chrome", + Version: "130.0.6723.117", + Source: "apps", + } + ghCli := fleet.Software{ + Name: "gh", + Source: "homebrew_packages", + } + + // Update the host's software. + software := []fleet.Software{ + safariApp, googleChromeApp, ghCli, + } + hostSoftware, err := s.ds.UpdateHostSoftware(context.Background(), host.ID, software) + require.NoError(t, err) + require.Len(t, hostSoftware.CurrInstalled(), 3) + + // Update the host's software installed paths for the software above. + // Google Chrome.app will have two installed paths one with team identifier set + // the other one set to empty. + swPaths := map[string]struct{}{} + for _, s := range software { + pathItems := [][2]string{{fmt.Sprintf("/some/path/%s", s.Name), ""}} + if s.Name == "Google Chrome.app" { + pathItems = [][2]string{ + {fmt.Sprintf("/some/path/%s", s.Name), "EQHXZ8M8AV"}, + {fmt.Sprintf("/some/other/path/%s", s.Name), ""}, + } + } + for _, pathItem := range pathItems { + path := pathItem[0] + teamIdentifier := pathItem[1] + key := fmt.Sprintf( + "%s%s%s%s%s", + path, fleet.SoftwareFieldSeparator, teamIdentifier, fleet.SoftwareFieldSeparator, s.ToUniqueStr(), + ) + swPaths[key] = struct{}{} + } + } + err = s.ds.UpdateHostSoftwareInstalledPaths(ctx, host.ID, swPaths, hostSoftware) + require.NoError(t, err) + + hostsCountTs := time.Now().UTC() + err = s.ds.SyncHostsSoftware(context.Background(), hostsCountTs) + require.NoError(t, err) + + getHostSoftwareResp := getHostSoftwareResponse{} + s.DoJSON("GET", fmt.Sprintf("/api/latest/fleet/hosts/%d/software", host.ID), + nil, http.StatusOK, &getHostSoftwareResp, + "per_page", "5", "page", "0", "order_key", "name", "order_direction", "desc", + ) + require.Len(t, getHostSoftwareResp.Software, 3) + require.Equal(t, "Safari.app", getHostSoftwareResp.Software[0].Name) + require.Len(t, getHostSoftwareResp.Software[0].InstalledVersions, 1) + require.Len(t, getHostSoftwareResp.Software[0].InstalledVersions[0].InstalledPaths, 1) + require.Equal(t, "/some/path/Safari.app", getHostSoftwareResp.Software[0].InstalledVersions[0].InstalledPaths[0]) + require.Len(t, getHostSoftwareResp.Software[0].InstalledVersions[0].SignatureInformation, 1) + require.Equal(t, "/some/path/Safari.app", getHostSoftwareResp.Software[0].InstalledVersions[0].SignatureInformation[0].InstalledPath) + require.Empty(t, getHostSoftwareResp.Software[0].InstalledVersions[0].SignatureInformation[0].TeamIdentifier) + + require.Equal(t, "Google Chrome.app", getHostSoftwareResp.Software[1].Name) + require.Len(t, getHostSoftwareResp.Software[1].InstalledVersions, 1) + require.Len(t, getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths, 2) + sort.Slice(getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths, func(i, j int) bool { + return getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths[i] < getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths[j] + }) + require.Equal(t, "/some/other/path/Google Chrome.app", getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths[0]) + require.Equal(t, "/some/path/Google Chrome.app", getHostSoftwareResp.Software[1].InstalledVersions[0].InstalledPaths[1]) + require.Len(t, getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation, 2) + sort.Slice(getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation, func(i, j int) bool { + return getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[i].InstalledPath < getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[j].InstalledPath + }) + require.Equal(t, "/some/other/path/Google Chrome.app", getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[0].InstalledPath) + require.Equal(t, "", getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[0].TeamIdentifier) + require.Equal(t, "/some/path/Google Chrome.app", getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[1].InstalledPath) + require.Equal(t, "EQHXZ8M8AV", getHostSoftwareResp.Software[1].InstalledVersions[0].SignatureInformation[1].TeamIdentifier) + + require.Equal(t, "gh", getHostSoftwareResp.Software[2].Name) + require.Len(t, getHostSoftwareResp.Software[2].InstalledVersions, 1) + require.Len(t, getHostSoftwareResp.Software[2].InstalledVersions[0].InstalledPaths, 1) + require.Equal(t, "/some/path/gh", getHostSoftwareResp.Software[2].InstalledVersions[0].InstalledPaths[0]) + require.Nil(t, getHostSoftwareResp.Software[2].InstalledVersions[0].SignatureInformation) +} diff --git a/server/service/osquery.go b/server/service/osquery.go index b2259aeb0858..21555df37270 100644 --- a/server/service/osquery.go +++ b/server/service/osquery.go @@ -1240,7 +1240,6 @@ func preProcessSoftwareResults( overrides map[string]osquery_utils.DetailQuery, logger log.Logger, ) { - // vsCodeExtensionsExtraQuery := hostDetailQueryPrefix + "software_vscode_extensions" preProcessSoftwareExtraResults(vsCodeExtensionsExtraQuery, host.ID, results, statuses, messages, osquery_utils.DetailQuery{}, logger) @@ -1377,9 +1376,12 @@ func preProcessSoftwareExtraResults( // Do not append results if the main query failed to run. continue } - (*results)[query] = removeOverrides((*results)[query], override) - - (*results)[query] = append((*results)[query], softwareExtraRows...) + if override.SoftwareProcessResults != nil { + (*results)[query] = override.SoftwareProcessResults((*results)[query], softwareExtraRows) + } else { + (*results)[query] = removeOverrides((*results)[query], override) + (*results)[query] = append((*results)[query], softwareExtraRows...) + } return } } diff --git a/server/service/osquery_test.go b/server/service/osquery_test.go index 286b6b8a28aa..5be2974a12ae 100644 --- a/server/service/osquery_test.go +++ b/server/service/osquery_test.go @@ -1062,6 +1062,7 @@ func verifyDiscovery(t *testing.T, queries, discovery map[string]string) { hostDetailQueryPrefix + "software_vscode_extensions": {}, hostDetailQueryPrefix + "software_macos_firefox": {}, hostDetailQueryPrefix + "battery": {}, + hostDetailQueryPrefix + "software_macos_codesign": {}, } for name := range queries { require.NotEmpty(t, discovery[name]) diff --git a/server/service/osquery_utils/queries.go b/server/service/osquery_utils/queries.go index 5797dd969705..da62dfef17a6 100644 --- a/server/service/osquery_utils/queries.go +++ b/server/service/osquery_utils/queries.go @@ -44,10 +44,13 @@ type DetailQuery struct { // empty, run on all platforms. Platforms []string // SoftwareOverrideMatch is a function that can be used to override a software - // result. The function evaluates a software detail query result row and deletes + // result. The function evaluates a software detail query result row and deletes // the result if the function returns true so the result of this detail query can be // used instead. SoftwareOverrideMatch func(row map[string]string) bool + // SoftwareProcessResults is a function that can be used to process entries of the main + // software query and append or modify data using results of additional queries. + SoftwareProcessResults func(mainSoftwareResults []map[string]string, additionalSoftwareResults []map[string]string) []map[string]string // IngestFunc translates a query result into an update to the host struct, // around data that lives on the hosts table. IngestFunc func(ctx context.Context, logger log.Logger, host *fleet.Host, rows []map[string]string) error @@ -820,7 +823,6 @@ var softwareMacOS = DetailQuery{ SELECT name AS name, COALESCE(NULLIF(bundle_short_version, ''), bundle_version) AS version, - 'Application (macOS)' AS type, bundle_identifier AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -833,7 +835,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -846,7 +847,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, '' AS bundle_identifier, identifier AS extension_id, browser_type AS browser, @@ -859,7 +859,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, '' AS bundle_identifier, identifier AS extension_id, 'firefox' AS browser, @@ -872,7 +871,6 @@ UNION SELECT name As name, version AS version, - 'Browser plugin (Safari)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -885,7 +883,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Homebrew)' AS type, '' AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -908,7 +905,6 @@ var softwareVSCodeExtensions = DetailQuery{ SELECT name, version, - 'IDE extension (VS Code)' AS type, '' AS bundle_identifier, uuid AS extension_id, '' AS browser, @@ -937,7 +933,6 @@ var softwareLinux = DetailQuery{ SELECT name AS name, version AS version, - 'Package (deb)' AS type, '' AS extension_id, '' AS browser, 'deb_packages' AS source, @@ -951,7 +946,6 @@ UNION SELECT package AS name, version AS version, - 'Package (Portage)' AS type, '' AS extension_id, '' AS browser, 'portage_packages' AS source, @@ -964,7 +958,6 @@ UNION SELECT name AS name, version AS version, - 'Package (RPM)' AS type, '' AS extension_id, '' AS browser, 'rpm_packages' AS source, @@ -977,7 +970,6 @@ UNION SELECT name AS name, version AS version, - 'Package (NPM)' AS type, '' AS extension_id, '' AS browser, 'npm_packages' AS source, @@ -990,7 +982,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, identifier AS extension_id, browser_type AS browser, 'chrome_extensions' AS source, @@ -1003,7 +994,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, identifier AS extension_id, 'firefox' AS browser, 'firefox_addons' AS source, @@ -1016,7 +1006,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS extension_id, '' AS browser, 'python_packages' AS source, @@ -1035,7 +1024,6 @@ var softwareWindows = DetailQuery{ SELECT name AS name, version AS version, - 'Program (Windows)' AS type, '' AS extension_id, '' AS browser, 'programs' AS source, @@ -1046,7 +1034,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Python)' AS type, '' AS extension_id, '' AS browser, 'python_packages' AS source, @@ -1057,7 +1044,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (IE)' AS type, '' AS extension_id, '' AS browser, 'ie_extensions' AS source, @@ -1068,7 +1054,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Chrome)' AS type, identifier AS extension_id, browser_type AS browser, 'chrome_extensions' AS source, @@ -1079,7 +1064,6 @@ UNION SELECT name AS name, version AS version, - 'Browser plugin (Firefox)' AS type, identifier AS extension_id, 'firefox' AS browser, 'firefox_addons' AS source, @@ -1090,7 +1074,6 @@ UNION SELECT name AS name, version AS version, - 'Package (Chocolatey)' AS type, '' AS extension_id, '' AS browser, 'chocolatey_packages' AS source, @@ -1108,7 +1091,6 @@ var softwareChrome = DetailQuery{ version AS version, identifier AS extension_id, browser_type AS browser, - 'Browser plugin (Chrome)' AS type, 'chrome_extensions' AS source, '' AS vendor, '' AS installed_path @@ -1123,10 +1105,13 @@ FROM chrome_extensions`, // Software queries expect specific columns to be present. Reference the // software_{macos|windows|linux} queries for the expected columns. var SoftwareOverrideQueries = map[string]DetailQuery{ - // macos_firefox Differentiates between Firefox and Firefox ESR by checking the RemotingName value in the + // macos_firefox differentiates between Firefox and Firefox ESR by checking the RemotingName value in the // application.ini file. If the RemotingName is 'firefox-esr', the name is set to 'Firefox ESR.app'. + // + // NOTE(lucas): This could be re-written to use SoftwareProcessResults so that this query doesn't need to match + // the columns of the main softwareMacOS query. "macos_firefox": { - Description: "A software override query[^1] to differentiate between Firefox and Firefox ESR on macOS. Requires `fleetd`", + Description: "A software override query[^1] to differentiate between Firefox and Firefox ESR on macOS. Requires `fleetd`", Query: ` WITH app_paths AS ( SELECT path @@ -1145,7 +1130,6 @@ var SoftwareOverrideQueries = map[string]DetailQuery{ ELSE 'Firefox.app' END AS name, COALESCE(NULLIF(apps.bundle_short_version, ''), apps.bundle_version) AS version, - 'Application (macOS)' AS type, apps.bundle_identifier AS bundle_identifier, '' AS extension_id, '' AS browser, @@ -1165,6 +1149,40 @@ var SoftwareOverrideQueries = map[string]DetailQuery{ return row["bundle_identifier"] == "org.mozilla.firefox" }, }, + // macos_codesign collects code signature information of apps on a separate query for two reasons: + // - codesign is a fleetd table (not part of osquery core). + // - Avoid growing the main `software_macos` query + // (having big queries can cause performance issues or be denylisted). + "macos_codesign": { + Query: ` + SELECT a.path, c.team_identifier + FROM apps a + JOIN codesign c ON a.path = c.path + `, + Description: "A software override query[^1] to append codesign information to macOS software entries. Requires `fleetd`", + Platforms: []string{"darwin"}, + Discovery: discoveryTable("codesign"), + SoftwareProcessResults: func(mainSoftwareResults, codesignResults []map[string]string) []map[string]string { + codesignInformation := make(map[string]string) // path -> team_identifier + for _, codesignResult := range codesignResults { + codesignInformation[codesignResult["path"]] = codesignResult["team_identifier"] + } + if len(codesignInformation) == 0 { + return mainSoftwareResults + } + + for _, result := range mainSoftwareResults { + codesignInfo := codesignInformation[result["installed_path"]] + if codesignInfo == "" { + // No codesign information for this application. + continue + } + result["team_identifier"] = codesignInfo + } + + return mainSoftwareResults + }, + }, } var usersQuery = DetailQuery{ @@ -1546,7 +1564,18 @@ func directIngestSoftware(ctx context.Context, logger log.Logger, host *fleet.Ho // NOTE: osquery is sometimes incorrectly returning the value "null" for some install paths. // Thus, we explicitly ignore such value here. strings.ToLower(installedPath) != "null" { - key := fmt.Sprintf("%s%s%s", installedPath, fleet.SoftwareFieldSeparator, s.ToUniqueStr()) + truncateString := func(str string, length int) string { + runes := []rune(str) + if len(runes) > length { + return string(runes[:length]) + } + return str + } + teamIdentifier := truncateString(row["team_identifier"], fleet.SoftwareTeamIdentifierMaxLength) + key := fmt.Sprintf( + "%s%s%s%s%s", + installedPath, fleet.SoftwareFieldSeparator, teamIdentifier, fleet.SoftwareFieldSeparator, s.ToUniqueStr(), + ) sPaths[key] = struct{}{} } } diff --git a/server/service/osquery_utils/queries_test.go b/server/service/osquery_utils/queries_test.go index 08ee059ac1d8..b15590a574f2 100644 --- a/server/service/osquery_utils/queries_test.go +++ b/server/service/osquery_utils/queries_test.go @@ -307,7 +307,7 @@ func TestGetDetailQueries(t *testing.T) { queriesWithUsersAndSoftware := GetDetailQueries(context.Background(), config.FleetConfig{App: config.AppConfig{EnableScheduledQueryStats: true}}, nil, &fleet.Features{EnableHostUsers: true, EnableSoftwareInventory: true}) qs = baseQueries qs = append(qs, "users", "users_chrome", "software_macos", "software_linux", "software_windows", "software_vscode_extensions", - "software_chrome", "scheduled_query_stats", "software_macos_firefox") + "software_chrome", "scheduled_query_stats", "software_macos_firefox", "software_macos_codesign") require.Len(t, queriesWithUsersAndSoftware, len(qs)) sortedKeysCompare(t, queriesWithUsersAndSoftware, qs) @@ -1338,7 +1338,7 @@ func TestDirectIngestSoftware(t *testing.T) { require.True(t, ds.UpdateHostSoftwareFuncInvoked) require.Len(t, calledWith, 1) - require.Contains(t, strings.Join(maps.Keys(calledWith), " "), fmt.Sprintf("%s%s%s", data[1]["installed_path"], fleet.SoftwareFieldSeparator, data[1]["name"])) + require.Contains(t, strings.Join(maps.Keys(calledWith), " "), fmt.Sprintf("%s%s%s%s%s", data[1]["installed_path"], fleet.SoftwareFieldSeparator, "", fleet.SoftwareFieldSeparator, data[1]["name"])) ds.UpdateHostSoftwareInstalledPathsFuncInvoked = false })