[Observability Onboarding] Display next steps (elastic#197179)

## Summary

Display next steps after the auto-detect script completes.

## Screenshot

<img width="905" alt="Screenshot 2024-10-22 at 09 30 51"
src="https://github.com/user-attachments/assets/85108ea2-510a-46f7-8b79-135be91b586f">

x-pack/plugins/fleet/server/services/epm/package_service.mock.ts

@@ -16,7 +16,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
   installCustomIntegration: jest.fn(),
   fetchFindLatestPackage: jest.fn(),
   readBundledPackage: jest.fn(),
-  getAgentPolicyInputs: jest.fn(),
+  getAgentPolicyConfigYAML: jest.fn(),
   getPackage: jest.fn(),
   getPackageFieldsMetadata: jest.fn(),
   getPackages: jest.fn(),

x-pack/plugins/fleet/server/services/epm/package_service.ts

@@ -28,7 +28,6 @@ import type {
   InstallablePackage,
   Installation,
   RegistryPackage,
-  TemplateAgentPolicyInput,
 } from '../../types';
 
 import type { FleetAuthzRouteConfig } from '../security/types';
@@ -116,12 +115,12 @@ export interface PackageClient {
     prerelease?: false;
   }): Promise<PackageList>;
 
-  getAgentPolicyInputs(
+  getAgentPolicyConfigYAML(
     pkgName: string,
     pkgVersion?: string,
     prerelease?: false,
     ignoreUnverified?: boolean
-  ): Promise<TemplateAgentPolicyInput[]>;
+  ): Promise<string>;
 
   reinstallEsAssets(
     packageInfo: InstallablePackage,
@@ -284,7 +283,7 @@ class PackageClientImpl implements PackageClient {
     return generatePackageInfoFromArchiveBuffer(archiveBuffer, 'application/zip');
   }
 
-  public async getAgentPolicyInputs(
+  public async getAgentPolicyConfigYAML(
     pkgName: string,
     pkgVersion?: string,
     prerelease?: false,
@@ -298,16 +297,14 @@ class PackageClientImpl implements PackageClient {
       pkgVersion = pkg.version;
     }
 
-    const { inputs } = await getTemplateInputs(
+    return getTemplateInputs(
       this.internalSoClient,
       pkgName,
       pkgVersion,
-      'json',
+      'yml',
       prerelease,
       ignoreUnverified
     );
-
-    return inputs;
   }
 
   public async getPackage(

x-pack/plugins/observability_solution/observability_onboarding/public/assets/auto_detect.sh

@@ -105,6 +105,7 @@ elastic_agent_config_path="/opt/Elastic/Agent/elastic-agent.yml"
 elastic_agent_tmp_config_path="/tmp/elastic-agent-config.tar"
 integration_names=()
 integration_titles=()
+config_files_with_password=()
 
 OS="$(uname)"
 ARCH="$(uname -m)"
@@ -155,7 +156,7 @@ download_elastic_agent() {
   curl -L -O "$download_url" --silent --fail
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
+    printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent downloaded to $(pwd)/$elastic_agent_artifact_name.tar.gz"
     update_step_progress "ea-download" "complete"
   else
     update_step_progress "ea-download" "danger" "Failed to download Elastic Agent, see script output for error."
@@ -167,7 +168,7 @@ extract_elastic_agent() {
   tar -xzf "${elastic_agent_artifact_name}.tar.gz"
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Archive extracted"
+    printf "\e[32;1m✓\e[0m %s\n" "Archive extracted"
     update_step_progress "ea-extract" "complete"
   else
     update_step_progress "ea-extract" "danger" "Failed to extract Elastic Agent, see script output for error."
@@ -179,7 +180,7 @@ install_elastic_agent() {
   "./${elastic_agent_artifact_name}/elastic-agent" install -f -n >/dev/null
 
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
+    printf "\e[32;1m✓\e[0m %s\n" "Elastic Agent installed to $(dirname "$elastic_agent_config_path")"
     update_step_progress "ea-install" "complete"
   else
     update_step_progress "ea-install" "danger" "Failed to install Elastic Agent, see script output for error."
@@ -224,7 +225,7 @@ ensure_elastic_agent_healthy() {
 
 backup_elastic_agent_config() {
   if [ -f "$elastic_agent_config_path" ]; then
-    echo -e "\nExisting config found at $elastic_agent_config_path"
+    printf "\n%s \e[36m%s\e[0m\n" "Existing config found at" "$elastic_agent_config_path"
 
     printf "\n\e[1;36m?\e[0m \e[1m%s\e[0m \e[2m%s\e[0m" "Create backup and continue installation?" "[Y/n] (default: Yes): "
     read confirmation_reply
@@ -241,7 +242,7 @@ backup_elastic_agent_config() {
       fi
 
       if [ "$?" -eq 0 ]; then
-        printf "\n\e[1;32m✓\e[0m %s\n" "Backup saved to $backup_path"
+        printf "\n\e[32;1m✓\e[0m %s \e[36m%s\e[0m\n" "Backup saved to" "$backup_path"
       else
         update_step_progress "ea-config" "warning" "Failed to backup existing configuration"
         fail "Failed to backup existing config - Try manually creating a backup or delete your existing config before re-running this script"
@@ -278,7 +279,7 @@ install_integrations() {
     --output "$elastic_agent_tmp_config_path"
 
   if [ "$?" -eq 0 ]; then
-    printf "\n\e[1;32m✓\e[0m %s\n" "Integrations installed"
+    printf "\n\e[32;1m✓\e[0m %s\n" "Integrations installed"
   else
     update_step_progress "ea-config" "warning" "Failed to install integrations"
     fail "Failed to install integrations"
@@ -297,10 +298,15 @@ apply_elastic_agent_config() {
     # Replace placeholder with the Ingest API key
     sed -i='' "s/\${API_KEY}/$decoded_ingest_api_key/" "$elastic_agent_config_path"
   if [ "$?" -eq 0 ]; then
-    printf "\e[1;32m✓\e[0m %s\n" "Config written to:"
-    tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$' | while read -r file; do
-      echo "  - $(dirname "$elastic_agent_config_path")/$file"
-    done
+    printf "\e[32;1m✓\e[0m %s\n" "Config files written to:"
+    while IFS= read -r file; do
+      local path="$(dirname "$elastic_agent_config_path")/$file"
+      printf "  \e[36m%s\e[0m\n" "$path"
+      grep '<PASSWORD>' "$path" >/dev/null
+      if [ "$?" -eq 0 ]; then
+        config_files_with_password+=("$path")
+      fi
+    done < <(tar --list --file "$elastic_agent_tmp_config_path" | grep '\.yml$')
 
     update_step_progress "ea-config" "complete"
   else
@@ -585,4 +591,11 @@ printf "\n\e[1m%s\e[0m\n" "Waiting for healthy status..."
 wait_for_elastic_agent_status
 ensure_elastic_agent_healthy
 
-printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running. You can now go back to Kibana and check for incoming logs."
+printf "\n\e[32m%s\e[0m\n" "🎉 Elastic Agent is configured and running!"
+
+printf "\n\e[1m%s\e[0m\n" "Next steps:"
+printf "\n• %s\n" "Go back to Kibana and check for incoming data"
+for path in "${config_files_with_password[@]}"; do
+  printf "\n• %s:\n  \e[36m%s\e[0m\n" "Collect $(known_integration_title "$(basename "${path%.yml}")") metrics by adding your username and password to" "$path"
+done
+printf "\n• %s:\n  \e[36;4m%s\e[0m\n" "For information on other standalone integration setups, visit" "https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html"

x-pack/plugins/observability_solution/observability_onboarding/public/assets/integrations.conf

@@ -1,14 +1,14 @@
 [system]
-title=System Logs And Metrics
+title=System
 
 [nginx]
-title=Nginx Logs
+title=Nginx
 patterns=
     /var/log/nginx/access.log*
     /var/log/nginx/error.log*
 
 [apache]
-title=Apache Logs
+title=Apache
 patterns=
     /var/log/apache2/access.log*
     /var/log/apache2/other_vhosts_access.log*
@@ -17,48 +17,48 @@ patterns=
     /var/log/httpd/error_log*
 
 [docker]
-title=Docker Container Logs
+title=Docker
 patterns=
     /var/lib/docker/containers/*/*-json.log
     /var/run/docker.sock
 
 [mysql]
-title=MySQL Logs
+title=MySQL
 patterns=
     /var/log/mysql/*error.log*
     /var/log/mysqld.log*
     /var/log/mysql/*-slow.log*
     /var/lib/mysql/*-slow.log*
 
 [postgresql]
-title=PostgreSQL Logs
+title=PostgreSQL
 patterns=
     /var/log/postgresql/postgresql-*-*.log*
     /*/postgresql-logs/*.log
     /etc/postgresql/*/main/postgresql.conf
     /var/log/postgresql/postgresql-*-*.csv*
 
 [redis]
-title=Redis Logs
+title=Redis
 patterns=
     /var/log/redis/redis-server.log*
     /etc/redis/redis.conf
 
 [haproxy]
-title=HAProxy Logs
+title=HAProxy
 patterns=
     /var/log/haproxy.log
     /etc/haproxy/haproxy.cfg
 
 [rabbitmq]
-title=RabbitMQ Logs
+title=RabbitMQ
 patterns=
     /var/log/rabbitmq/rabbit@*.log
     /etc/rabbitmq/rabbitmq.conf
     /etc/rabbitmq/rabbitmq.config
 
 [kafka]
-title=Kafka Logs
+title=Kafka
 patterns=
     /var/log/kafka/server.log
     /etc/kafka/server.properties
@@ -68,19 +68,19 @@ patterns=
     /*/logs/kafka-*.log*
 
 [mongodb]
-title=MongoDB Logs
+title=MongoDB
 patterns=
     /var/log/mongodb/mongod.log
 
 [apache_tomcat]
-title=Apache Tomcat Logs
+title=Apache Tomcat
 patterns=
     /opt/tomcat/logs/localhost_access_log.*.txt
     /opt/tomcat/logs/catalina.*.log
     /opt/tomcat/logs/localhost.*.log
 
 [prometheus]
-title=Prometheus Server overview
+title=Prometheus
 patterns=
     /var/log/prometheus/prometheus.log
     /etc/prometheus/prometheus.yml

x-pack/plugins/observability_solution/observability_onboarding/server/routes/flow/route.ts

@@ -263,8 +263,8 @@ const createFlowRoute = createObservabilityOnboardingServerRoute({
  *
  * The request format is TSV (tab-separated values) to simplify parsing in bash.
  *
- * The response format is either a YAML file or a tar archive containing the Elastic Agent
- * configuration, depending on the `Accept` header.
+ * The response format is a tar archive containing the Elastic Agent configuration, depending on the
+ * `Accept` header.
  *
  * Errors during installation are ignore unless all integrations fail to install. When that happens
  * a 500 Internal Server Error is returned with the first error message.
@@ -348,7 +348,7 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
         }
         return acc;
       }, []);
-      // Errors during installation are ignore unless all integrations fail to install. When that happens
+      // Errors during installation are ignored unless all integrations fail to install. When that happens
       // a 500 Internal Server Error is returned with the first error message.
       if (!installedIntegrations.length) {
         throw (settledResults[0] as PromiseRejectedResult).reason;
@@ -383,20 +383,11 @@ const integrationsInstallRoute = createObservabilityOnboardingServerRoute({
       ? [plugins.cloud?.setup?.elasticsearchUrl]
       : await getFallbackESUrl(services.esLegacyConfigService);
 
-    if (request.headers.accept === 'application/x-tar') {
-      return response.ok({
-        headers: {
-          'content-type': 'application/x-tar',
-        },
-        body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
-      });
-    }
-
     return response.ok({
       headers: {
-        'content-type': 'application/yaml',
+        'content-type': 'application/x-tar',
       },
-      body: generateAgentConfigYAML({ elasticsearchUrl, installedIntegrations }),
+      body: generateAgentConfigTar({ elasticsearchUrl, installedIntegrations }),
     });
   },
 });
@@ -423,15 +414,15 @@ async function ensureInstalledIntegrations(
       if (installSource === 'registry') {
         const installation = await packageClient.ensureInstalledPackage({ pkgName });
         const pkg = installation.package;
-        const inputs = await packageClient.getAgentPolicyInputs(pkg.name, pkg.version);
+        const config = await packageClient.getAgentPolicyConfigYAML(pkg.name, pkg.version);
         const { packageInfo } = await packageClient.getPackage(pkg.name, pkg.version);
 
         return {
           installSource,
           pkgName: pkg.name,
           pkgVersion: pkg.version,
           title: packageInfo.title,
-          inputs: inputs.filter((input) => input.type !== 'httpjson'),
+          config,
           dataStreams:
             packageInfo.data_streams?.map(({ type, dataset }) => ({ type, dataset })) ?? [],
           kibanaAssets: pkg.installed_kibana,
@@ -447,19 +438,21 @@ async function ensureInstalledIntegrations(
         pkgName,
         pkgVersion: '1.0.0', // Custom integrations are always installed as version `1.0.0`
         title: pkgName,
-        inputs: [
-          {
-            id: `filestream-${pkgName}`,
-            type: 'filestream',
-            streams: [
-              {
-                id: `filestream-${pkgName}`,
-                data_stream: dataStream,
-                paths: integration.logFilePaths,
-              },
-            ],
-          },
-        ],
+        config: dump({
+          inputs: [
+            {
+              id: `filestream-${pkgName}`,
+              type: 'filestream',
+              streams: [
+                {
+                  id: `filestream-${pkgName}`,
+                  data_stream: dataStream,
+                  paths: integration.logFilePaths,
+                },
+              ],
+            },
+          ],
+        }),
         dataStreams: [dataStream],
         kibanaAssets: [],
       };
@@ -538,25 +531,6 @@ function parseIntegrationsTSV(tsv: string) {
   );
 }
 
-const generateAgentConfigYAML = ({
-  elasticsearchUrl,
-  installedIntegrations,
-}: {
-  elasticsearchUrl: string[];
-  installedIntegrations: InstalledIntegration[];
-}) => {
-  return dump({
-    outputs: {
-      default: {
-        type: 'elasticsearch',
-        hosts: elasticsearchUrl,
-        api_key: '${API_KEY}', // Placeholder to be replaced by bash script with the actual API key
-      },
-    },
-    inputs: installedIntegrations.map(({ inputs }) => inputs).flat(),
-  });
-};
-
 const generateAgentConfigTar = ({
   elasticsearchUrl,
   installedIntegrations,
@@ -592,7 +566,7 @@ const generateAgentConfigTar = ({
       path: `inputs.d/${integration.pkgName}.yml`,
       mode: 0o644,
       mtime: now,
-      data: dump({ inputs: integration.inputs }),
+      data: integration.config,
     })),
   ]);
 };

x-pack/plugins/observability_solution/observability_onboarding/server/routes/types.ts

@@ -57,7 +57,7 @@ export const IntegrationRT = t.type({
   pkgName: t.string,
   pkgVersion: t.string,
   title: t.string,
-  inputs: t.array(t.unknown),
+  config: t.string,
   dataStreams: t.array(
     t.type({
       type: t.string,