Telemetry: collect components status (elastic#163912)

## Summary Querying agent components status to add to telemetry. Depends on elastic/elasticsearch#98471 ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>
rshen91 · Aug 15, 2023 · 396d04a · 396d04a
1 parent f4381ac
commit 396d04a
Show file tree

Hide file tree

Showing 4 changed files with 126 additions and 25 deletions.
diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts
@@ -75,13 +75,19 @@ export interface AgentData {
     version: string;
     count: number;
   }>;
+  components_status: Array<{
+    id: string;
+    status: string;
+    count: number;
+  }>;
 }
 
 const DEFAULT_AGENT_DATA = {
   agent_checkin_status: { error: 0, degraded: 0 },
   agents_per_policy: [],
   agents_per_version: [],
   agents_per_os: [],
+  components_status: [],
 };
 
 export const getAgentData = async (
@@ -135,6 +141,25 @@ export const getAgentData = async (
               ],
             },
           },
+          components: {
+            nested: {
+              path: 'components',
+            },
+            aggs: {
+              components_status: {
+                multi_terms: {
+                  terms: [
+                    {
+                      field: 'components.id',
+                    },
+                    {
+                      field: 'components.status',
+                    },
+                  ],
+                },
+              },
+            },
+          },
         },
       },
       { signal: abortController.signal }
@@ -190,11 +215,20 @@ export const getAgentData = async (
       count: bucket.doc_count,
     }));
 
+    const componentsStatus = (
+      (response?.aggregations?.components as any).components_status?.buckets ?? []
+    ).map((bucket: any) => ({
+      id: bucket.key[0],
+      status: bucket.key[1],
+      count: bucket.doc_count,
+    }));
+
     return {
       agent_checkin_status: statuses,
       agents_per_policy: agentsPerPolicy,
       agents_per_version: agentsPerVersion,
       agents_per_os: agentsPerOS,
+      components_status: componentsStatus,
     };
   } catch (error) {
     if (error.statusCode === 404) {

diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts
@@ -12,7 +12,7 @@ import type { FleetConfigType } from '..';
 
 import { getIsAgentsEnabled } from './config_collectors';
 import { getAgentUsage, getAgentData } from './agent_collectors';
-import type { AgentUsage } from './agent_collectors';
+import type { AgentUsage, AgentData } from './agent_collectors';
 import { getInternalClients } from './helpers';
 import { getPackageUsage } from './package_collectors';
 import type { PackageUsage } from './package_collectors';
@@ -30,18 +30,9 @@ export interface Usage {
   fleet_server: FleetServerUsage;
 }
 
-export interface FleetUsage extends Usage {
+export interface FleetUsage extends Usage, AgentData {
   fleet_server_config: { policies: Array<{ input_config: any }> };
   agent_policies: { count: number; output_types: string[] };
-  agents_per_version: Array<{
-    version: string;
-    count: number;
-  }>;
-  agent_checkin_status: {
-    error: number;
-    degraded: number;
-  };
-  agents_per_policy: number[];
   agent_logs_panics_last_hour: AgentPanicLogsData['agent_logs_panics_last_hour'];
   agent_logs_top_errors?: string[];
   fleet_server_logs_top_errors?: string[];

diff --git a/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts b/x-pack/plugins/fleet/server/integration_tests/fleet_usage_telemetry.test.ts
@@ -136,6 +136,16 @@ describe('fleet usage telemetry', () => {
               version: '22.04.2 LTS (Jammy Jellyfish)',
             },
           },
+          components: [
+            {
+              id: 'filestream-monitoring',
+              status: 'UNHEALTHY',
+            },
+            {
+              id: 'beat/metrics-monitoring',
+              status: 'HEALTHY',
+            },
+          ],
         },
         {
           create: {
@@ -156,6 +166,16 @@ describe('fleet usage telemetry', () => {
               version: '20.04.5 LTS (Focal Fossa)',
             },
           },
+          components: [
+            {
+              id: 'filestream-monitoring',
+              status: 'HEALTHY',
+            },
+            {
+              id: 'beat/metrics-monitoring',
+              status: 'HEALTHY',
+            },
+          ],
         },
         {
           create: {
@@ -176,6 +196,16 @@ describe('fleet usage telemetry', () => {
               version: '20.04.5 LTS (Focal Fossa)',
             },
           },
+          components: [
+            {
+              id: 'filestream-monitoring',
+              status: 'HEALTHY',
+            },
+            {
+              id: 'beat/metrics-monitoring',
+              status: 'HEALTHY',
+            },
+          ],
         },
       ],
       refresh: 'wait_for',
@@ -404,6 +434,24 @@ describe('fleet usage telemetry', () => {
             count: 1,
           },
         ],
+        components_status: [
+          /* To uncomment when ES new snapshot will be built 
+          {
+            id: 'filestream-monitoring',
+            status: 'HEALTHY',
+            count: 1,
+          },
+          {
+            id: 'filestream-monitoring',
+            status: 'UNHEALTHY',
+            count: 1,
+          },
+          {
+            id: 'beat/metrics-monitoring',
+            status: 'HEALTHY',
+            count: 2,
+          }, */
+        ],
         fleet_server_config: {
           policies: [
             {

diff --git a/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts b/x-pack/plugins/fleet/server/services/telemetry/fleet_usages_schema.ts
@@ -262,23 +262,51 @@ export const fleetUsagesSchema: RootSchema<any> = {
     },
   },
   agents_per_os: {
-    properties: {
-      name: {
-        type: 'keyword',
-        _meta: {
-          description: 'Agent OS enrolled to this kibana',
+    type: 'array',
+    items: {
+      properties: {
+        name: {
+          type: 'keyword',
+          _meta: {
+            description: 'Agent OS enrolled to this kibana',
+          },
         },
-      },
-      version: {
-        type: 'keyword',
-        _meta: {
-          description: 'Agent OS version enrolled to this kibana',
+        version: {
+          type: 'keyword',
+          _meta: {
+            description: 'Agent OS version enrolled to this kibana',
+          },
+        },
+        count: {
+          type: 'long',
+          _meta: {
+            description: 'Number of agents enrolled that use this OS',
+          },
         },
       },
-      count: {
-        type: 'long',
-        _meta: {
-          description: 'Number of agents enrolled that use this OS',
+    },
+  },
+  components_status: {
+    type: 'array',
+    items: {
+      properties: {
+        id: {
+          type: 'keyword',
+          _meta: {
+            description: 'Component Id',
+          },
+        },
+        status: {
+          type: 'keyword',
+          _meta: {
+            description: 'Component Status',
+          },
+        },
+        count: {
+          type: 'long',
+          _meta: {
+            description: 'Number of this component with this status',
+          },
         },
       },
     },