[Fleet] Move Fleet Setup to start lifecycle (#117552) (#118554)

* Call setup on fleet start, remove API calls

* Fix unused import

* Revert removal of setup API call

* Restructor fleetSetupCompleted promise

* Add logging + handle setup failures

* Restructure logging to mix of debug/info

* Maybe fix failing tests

* Try fixing tests again

* Fix another dashboard test

* Re-add output logs after merge

* Log non-fatal errors during Fleet setup on boot

* Don't rely on fleetSetupCompleted to be called

* Fix failing test

* Track fleet setup status to avoid double calls

* Use IIFE in place of Promise ctor

* Remove unnecessary fleetSetupStatus value

* Move non-error logs into setupFleet method

* Remove unused formatNonFatalErrors import

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kyle Pollich <kyle.pollich@elastic.co>

Author: kibanamachine

test/accessibility/apps/dashboard.ts

@@ -45,6 +45,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     it('add a visualization', async () => {
+      await testSubjects.setValue('savedObjectFinderSearchInput', '[Flights]');
       await testSubjects.click('savedObjectTitle[Flights]-Delay-Buckets');
       await a11y.testAppSnapshot();
     });
@@ -85,6 +86,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
     });
 
     it('Add one more saved object to cancel it', async () => {
+      await testSubjects.setValue('savedObjectFinderSearchInput', '[Flights]');
       await testSubjects.click('savedObjectTitle[Flights]-Destination-Weather');
       await a11y.testAppSnapshot();
     });

test/examples/embeddables/adding_children.ts

@@ -23,6 +23,8 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
       await testSubjects.click('embeddablePanelToggleMenuIcon');
       await testSubjects.click('embeddablePanelAction-ACTION_ADD_PANEL');
       await testSubjects.waitForDeleted('savedObjectFinderLoadingIndicator');
+      await testSubjects.click('savedObjectFinderFilterButton');
+      await testSubjects.click('savedObjectFinderFilter-todo');
       await testSubjects.click('savedObjectTitleGarbage');
       await testSubjects.moveMouseTo('euiFlyoutCloseButton');
       await flyout.ensureClosed('dashboardAddPanel');

test/functional/apps/management/_mgmt_import_saved_objects.js

@@ -39,6 +39,7 @@ export default function ({ getService, getPageObjects }) {
       await PageObjects.savedObjects.clickConfirmChanges();
       await PageObjects.savedObjects.clickImportDone();
       await PageObjects.savedObjects.waitTableIsLoaded();
+      await PageObjects.savedObjects.searchForObject('mysaved');
 
       //instead of asserting on count- am asserting on the titles- which is more accurate than count.
       const objects = await PageObjects.savedObjects.getRowTitles();

x-pack/plugins/fleet/server/plugin.ts

@@ -20,7 +20,7 @@ import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 
 import type { TelemetryPluginSetup, TelemetryPluginStart } from 'src/plugins/telemetry/server';
 
-import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
+import { DEFAULT_APP_CATEGORIES, SavedObjectsClient } from '../../../../src/core/server';
 import type { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server';
 import type { LicensingPluginSetup, ILicense } from '../../licensing/server';
 import type {
@@ -83,6 +83,7 @@ import { RouterWrappers } from './routes/security';
 import { FleetArtifactsClient } from './services/artifacts';
 import type { FleetRouter } from './types/request_context';
 import { TelemetryEventsSender } from './telemetry/sender';
+import { setupFleet } from './services/setup';
 
 export interface FleetSetupDeps {
   licensing: LicensingPluginSetup;
@@ -332,8 +333,22 @@ export class FleetPlugin
 
     this.telemetryEventsSender.start(plugins.telemetry, core);
 
+    const logger = appContextService.getLogger();
+
+    const fleetSetupPromise = (async () => {
+      try {
+        await setupFleet(
+          new SavedObjectsClient(core.savedObjects.createInternalRepository()),
+          core.elasticsearch.client.asInternalUser
+        );
+      } catch (error) {
+        logger.warn('Fleet setup failed');
+        logger.warn(error);
+      }
+    })();
+
     return {
-      fleetSetupCompleted: () => Promise.resolve(),
+      fleetSetupCompleted: () => fleetSetupPromise,
       esIndexPatternService: new ESIndexPatternSavedObjectService(),
       packageService: {
         getInstallation,

x-pack/plugins/fleet/server/routes/setup/handlers.test.ts

@@ -18,6 +18,7 @@ import { fleetSetupHandler } from './handlers';
 
 jest.mock('../../services/setup', () => {
   return {
+    ...jest.requireActual('../../services/setup'),
     setupFleet: jest.fn(),
   };
 });

x-pack/plugins/fleet/server/routes/setup/handlers.ts

@@ -7,7 +7,7 @@
 
 import { appContextService } from '../../services';
 import type { GetFleetStatusResponse, PostFleetSetupResponse } from '../../../common';
-import { setupFleet } from '../../services/setup';
+import { formatNonFatalErrors, setupFleet } from '../../services/setup';
 import { hasFleetServers } from '../../services/fleet_server';
 import { defaultIngestErrorHandler } from '../../errors';
 import type { FleetRequestHandler } from '../../types';
@@ -50,24 +50,8 @@ export const fleetSetupHandler: FleetRequestHandler = async (context, request, r
     const setupStatus = await setupFleet(soClient, esClient);
     const body: PostFleetSetupResponse = {
       ...setupStatus,
-      nonFatalErrors: setupStatus.nonFatalErrors.flatMap((e) => {
-        // JSONify the error object so it can be displayed properly in the UI
-        if ('error' in e) {
-          return {
-            name: e.error.name,
-            message: e.error.message,
-          };
-        } else {
-          return e.errors.map((upgradePackagePolicyError: any) => {
-            return {
-              name: upgradePackagePolicyError.key,
-              message: upgradePackagePolicyError.message,
-            };
-          });
-        }
-      }),
+      nonFatalErrors: formatNonFatalErrors(setupStatus.nonFatalErrors),
     };
-
     return response.ok({ body });
   } catch (error) {
     return defaultIngestErrorHandler({ error, response });

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -73,6 +73,8 @@ export async function ensurePreconfiguredOutputs(
   esClient: ElasticsearchClient,
   outputs: PreconfiguredOutput[]
 ) {
+  const logger = appContextService.getLogger();
+
   if (outputs.length === 0) {
     return;
   }
@@ -106,8 +108,10 @@ export async function ensurePreconfiguredOutputs(
         existingOutput && isPreconfiguredOutputDifferentFromCurrent(existingOutput, data);
 
       if (isCreate) {
+        logger.debug(`Creating output ${output.id}`);
         await outputService.create(soClient, data, { id, fromPreconfiguration: true });
       } else if (isUpdateWithNewData) {
+        logger.debug(`Updating output ${output.id}`);
         await outputService.update(soClient, id, data, { fromPreconfiguration: true });
         // Bump revision of all policies using that output
         if (outputData.is_default || outputData.is_default_monitoring) {
@@ -335,7 +339,7 @@ export async function ensurePreconfiguredPackagesAndPolicies(
         await soClient
           .delete(AGENT_POLICY_SAVED_OBJECT_TYPE, policy!.id)
           // swallow error
-          .catch((deleteErr) => appContextService.getLogger().error(deleteErr));
+          .catch((deleteErr) => logger.error(deleteErr));
 
         throw err;
       }

x-pack/plugins/fleet/server/services/setup.ts

@@ -51,6 +51,9 @@ async function createSetupSideEffects(
   soClient: SavedObjectsClientContract,
   esClient: ElasticsearchClient
 ): Promise<SetupStatus> {
+  const logger = appContextService.getLogger();
+  logger.info('Beginning fleet setup');
+
   const {
     agentPolicies: policiesOrUndefined,
     packages: packagesOrUndefined,
@@ -60,6 +63,7 @@ async function createSetupSideEffects(
   const policies = policiesOrUndefined ?? [];
   let packages = packagesOrUndefined ?? [];
 
+  logger.debug('Setting up Fleet outputs');
   await Promise.all([
     ensurePreconfiguredOutputs(soClient, esClient, outputsOrUndefined ?? []),
     settingsService.settingsSetup(soClient),
@@ -68,6 +72,7 @@ async function createSetupSideEffects(
   const defaultOutput = await outputService.ensureDefaultOutput(soClient);
 
   if (appContextService.getConfig()?.agentIdVerificationEnabled) {
+    logger.debug('Setting up Fleet Elasticsearch assets');
     await ensureFleetGlobalEsAssets(soClient, esClient);
   }
 
@@ -91,6 +96,8 @@ async function createSetupSideEffects(
     ...autoUpdateablePackages.filter((pkg) => !preconfiguredPackageNames.has(pkg.name)),
   ];
 
+  logger.debug('Setting up initial Fleet packages');
+
   const { nonFatalErrors } = await ensurePreconfiguredPackagesAndPolicies(
     soClient,
     esClient,
@@ -99,11 +106,22 @@ async function createSetupSideEffects(
     defaultOutput
   );
 
+  logger.debug('Cleaning up Fleet outputs');
   await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []);
 
+  logger.debug('Setting up Fleet enrollment keys');
   await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient);
+
+  logger.debug('Setting up Fleet Server agent policies');
   await ensureFleetServerAgentPoliciesExists(soClient, esClient);
 
+  if (nonFatalErrors.length > 0) {
+    logger.info('Encountered non fatal errors during Fleet setup');
+    formatNonFatalErrors(nonFatalErrors).forEach((error) => logger.info(JSON.stringify(error)));
+  }
+
+  logger.info('Fleet setup completed');
+
   return {
     isInitialized: true,
     nonFatalErrors,
@@ -119,6 +137,7 @@ export async function ensureFleetGlobalEsAssets(
 ) {
   const logger = appContextService.getLogger();
   // Ensure Global Fleet ES assets are installed
+  logger.debug('Creating Fleet component template and ingest pipeline');
   const globalAssetsRes = await Promise.all([
     ensureDefaultComponentTemplate(esClient),
     ensureFleetFinalPipelineIsInstalled(esClient),
@@ -141,7 +160,7 @@ export async function ensureFleetGlobalEsAssets(
           savedObjectsClient: soClient,
           pkgkey: pkgToPkgKey({ name: installation.name, version: installation.version }),
           esClient,
-          // Force install the pacakge will update the index template and the datastream write indices
+          // Force install the package will update the index template and the datastream write indices
           force: true,
         }).catch((err) => {
           logger.error(
@@ -187,3 +206,27 @@ export async function ensureDefaultEnrollmentAPIKeysExists(
     })
   );
 }
+
+/**
+ * Maps the `nonFatalErrors` object returned by the setup process to a more readable
+ * and predictable format suitable for logging output or UI presentation.
+ */
+export function formatNonFatalErrors(
+  nonFatalErrors: SetupStatus['nonFatalErrors']
+): Array<{ name: string; message: string }> {
+  return nonFatalErrors.flatMap((e) => {
+    if ('error' in e) {
+      return {
+        name: e.error.name,
+        message: e.error.message,
+      };
+    } else {
+      return e.errors.map((upgradePackagePolicyError: any) => {
+        return {
+          name: upgradePackagePolicyError.key,
+          message: upgradePackagePolicyError.message,
+        };
+      });
+    }
+  });
+}