diff --git a/x-pack/plugins/fleet/common/types/models/agent_policy.ts b/x-pack/plugins/fleet/common/types/models/agent_policy.ts index 919d67392770d6..a653bbfb8b2238 100644 --- a/x-pack/plugins/fleet/common/types/models/agent_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/agent_policy.ts @@ -5,6 +5,8 @@ * 2.0. */ +import type { SecurityRoleDescriptor } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + import type { agentPolicyStatuses } from '../../constants'; import type { MonitoringType, PolicySecretReference, ValueOf } from '..'; @@ -77,15 +79,7 @@ export interface FullAgentPolicyInput { [key: string]: any; } -export interface FullAgentPolicyOutputPermissions { - [packagePolicyName: string]: { - cluster?: string[]; - indices?: Array<{ - names: string[]; - privileges: string[]; - }>; - }; -} +export type FullAgentPolicyOutputPermissions = Record; export type FullAgentPolicyOutput = Pick & { proxy_url?: string; diff --git a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts index c6eaba98135f39..139f07fb999b39 100644 --- a/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/integration_tests/cloud_preconfiguration.test.ts @@ -312,31 +312,11 @@ describe('Fleet preconfiguration reset', () => { cluster: ['cluster:monitor/main'], indices: [ { - names: ['logs-apm.app-default'], + names: ['traces-*', 'logs-*', 'metrics-*'], privileges: ['auto_configure', 'create_doc'], }, { - names: ['metrics-apm.app.*-default'], - privileges: ['auto_configure', 'create_doc'], - }, - { - names: ['logs-apm.error-default'], - privileges: ['auto_configure', 'create_doc'], - }, - { - names: ['metrics-apm.internal-default'], - privileges: ['auto_configure', 'create_doc'], - }, - { - names: ['metrics-apm.profiling-default'], - privileges: ['auto_configure', 'create_doc'], - }, - { - names: ['traces-apm.rum-default'], - privileges: ['auto_configure', 'create_doc'], - }, - { - names: ['traces-apm.sampled-default'], + names: ['traces-apm.sampled-*'], privileges: [ 'auto_configure', 'create_doc', @@ -345,10 +325,6 @@ describe('Fleet preconfiguration reset', () => { 'read', ], }, - { - names: ['traces-apm-default'], - privileges: ['auto_configure', 'create_doc'], - }, ], }, }, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts index 8bd45d80d94297..e4f2b30bc4a9e1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.test.ts @@ -239,26 +239,66 @@ packageInfoCache.set('profiler_collector-8.9.0-preview', { }, }); +packageInfoCache.set('apm-8.9.0-preview', { + format_version: '2.7.0', + name: 'apm', + title: 'APM', + version: '8.9.0-preview', + license: 'basic', + description: 'APM Server integration', + type: 'integration', + release: 'beta', + categories: ['observability'], + icons: [], + owner: { github: 'elastic/apm-server' }, + data_streams: [], + latestVersion: '8.9.0-preview', + status: 'not_installed', + assets: { + kibana: { + csp_rule_template: [], + dashboard: [], + visualization: [], + search: [], + index_pattern: [], + map: [], + lens: [], + security_rule: [], + ml_module: [], + tag: [], + osquery_pack_asset: [], + osquery_saved_query: [], + }, + elasticsearch: { + component_template: [], + ingest_pipeline: [], + ilm_policy: [], + transform: [], + index_template: [], + data_stream_ilm_policy: [], + ml_model: [], + }, + }, +}); + describe('storedPackagePoliciesToAgentPermissions()', () => { it('Returns `undefined` if there are no package policies', async () => { const permissions = await storedPackagePoliciesToAgentPermissions(packageInfoCache, []); expect(permissions).toBeUndefined(); }); - it('Throw an error if package policies is not an array', async () => { - await expect(() => - storedPackagePoliciesToAgentPermissions(packageInfoCache, undefined) - ).rejects.toThrow( + it('Throw an error if package policies is not an array', () => { + expect(() => storedPackagePoliciesToAgentPermissions(packageInfoCache, undefined)).toThrow( /storedPackagePoliciesToAgentPermissions should be called with a PackagePolicy/ ); }); - it('Returns the default permissions if a package policy does not have a package', async () => { - await expect(() => + it('Returns the default permissions if a package policy does not have a package', () => { + expect(() => storedPackagePoliciesToAgentPermissions(packageInfoCache, [ { name: 'foo', package: undefined } as PackagePolicy, ]) - ).rejects.toThrow(/No package for package policy foo/); + ).toThrow(/No package for package policy foo/); }); it('Returns the permissions for the enabled inputs', async () => { @@ -545,6 +585,52 @@ describe('storedPackagePoliciesToAgentPermissions()', () => { }, }); }); + + it('returns the correct permissions for the APM package', async () => { + const packagePolicies: PackagePolicy[] = [ + { + id: 'package-policy-uuid-test-123', + name: 'test-policy', + namespace: '', + enabled: true, + package: { name: 'apm', version: '8.9.0-preview', title: 'Test Package' }, + inputs: [ + { + type: 'pf-elastic-collector', + enabled: true, + streams: [], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + }, + ]; + + const permissions = await storedPackagePoliciesToAgentPermissions( + packageInfoCache, + packagePolicies + ); + + expect(permissions).toMatchObject({ + 'package-policy-uuid-test-123': { + cluster: ['cluster:monitor/main'], + indices: [ + { + names: ['traces-*', 'logs-*', 'metrics-*'], + privileges: ['auto_configure', 'create_doc'], + }, + { + names: ['traces-apm.sampled-*'], + privileges: ['auto_configure', 'create_doc', 'maintenance', 'monitor', 'read'], + }, + ], + }, + }); + }); }); describe('getDataStreamPrivileges()', () => { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts index eeb81ca9c17240..4445ebbe847694 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_permissions.ts @@ -5,7 +5,13 @@ * 2.0. */ +import type { + SecurityIndicesPrivileges, + SecurityRoleDescriptor, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + import { + FLEET_APM_PACKAGE, FLEET_UNIVERSAL_PROFILING_COLLECTOR_PACKAGE, FLEET_UNIVERSAL_PROFILING_SYMBOLIZER_PACKAGE, } from '../../../common/constants'; @@ -34,10 +40,10 @@ export const UNIVERSAL_PROFILING_PERMISSIONS = [ 'view_index_metadata', ]; -export async function storedPackagePoliciesToAgentPermissions( +export function storedPackagePoliciesToAgentPermissions( packageInfoCache: Map, packagePolicies?: PackagePolicy[] -): Promise { +): FullAgentPolicyOutputPermissions | undefined { // I'm not sure what permissions to return for this case, so let's return the defaults if (!packagePolicies) { throw new Error( @@ -49,114 +55,116 @@ export async function storedPackagePoliciesToAgentPermissions( return; } - const permissionEntries = (packagePolicies as PackagePolicy[]).map>( - async (packagePolicy) => { - if (!packagePolicy.package) { - throw new Error(`No package for package policy ${packagePolicy.name ?? packagePolicy.id}`); - } - - const pkg = packageInfoCache.get(pkgToPkgKey(packagePolicy.package))!; - - // Special handling for Universal Profiling packages, as it does not use data streams _only_, - // but also indices that do not adhere to the convention. - if ( - pkg.name === FLEET_UNIVERSAL_PROFILING_SYMBOLIZER_PACKAGE || - pkg.name === FLEET_UNIVERSAL_PROFILING_COLLECTOR_PACKAGE - ) { - return Promise.resolve(universalProfilingPermissions(packagePolicy.id)); - } - - const dataStreams = getNormalizedDataStreams(pkg); - if (!dataStreams || dataStreams.length === 0) { - return [packagePolicy.name, undefined]; - } - - let dataStreamsForPermissions: DataStreamMeta[]; - - switch (pkg.name) { - case 'endpoint': - // - Endpoint doesn't store the `data_stream` metadata in - // `packagePolicy.inputs`, so we will use _all_ data_streams from the - // package. - dataStreamsForPermissions = dataStreams; - break; - - case 'apm': - // - APM doesn't store the `data_stream` metadata in - // `packagePolicy.inputs`, so we will use _all_ data_streams from - // the package. - dataStreamsForPermissions = dataStreams; - break; - - case 'osquery_manager': - // - Osquery manager doesn't store the `data_stream` metadata in - // `packagePolicy.inputs`, so we will use _all_ data_streams from - // the package. - dataStreamsForPermissions = dataStreams; - break; - - default: - // - Normal packages store some of the `data_stream` metadata in - // `packagePolicy.inputs[].streams[].data_stream` - // - The rest of the metadata needs to be fetched from the - // `data_stream` object in the package. The link is - // `packagePolicy.inputs[].type == dataStreams.streams[].input` - // - Some packages (custom logs) have a compiled dataset, stored in - // `input.streams.compiled_stream.data_stream.dataset` - dataStreamsForPermissions = packagePolicy.inputs - .filter((i) => i.enabled) - .flatMap((input) => { - if (!input.streams) { - return []; - } - - const dataStreams_: DataStreamMeta[] = []; - - input.streams - .filter((s) => s.enabled) - .forEach((stream) => { - if (!('data_stream' in stream)) { - return; - } - - const ds: DataStreamMeta = { - type: stream.data_stream.type, - dataset: - stream.compiled_stream?.data_stream?.dataset ?? stream.data_stream.dataset, - }; - - if (stream.data_stream.elasticsearch) { - ds.elasticsearch = stream.data_stream.elasticsearch; - } - - dataStreams_.push(ds); - }); - - return dataStreams_; - }); - } - - let clusterRoleDescriptor = {}; - const cluster = packagePolicy?.elasticsearch?.privileges?.cluster ?? []; - if (cluster.length > 0) { - clusterRoleDescriptor = { - cluster, - }; - } - - return [ - packagePolicy.id, - { - indices: dataStreamsForPermissions.map((ds) => - getDataStreamPrivileges(ds, packagePolicy.namespace) - ), - ...clusterRoleDescriptor, - }, - ]; + const permissionEntries = packagePolicies.map((packagePolicy) => { + if (!packagePolicy.package) { + throw new Error(`No package for package policy ${packagePolicy.name ?? packagePolicy.id}`); + } + + const pkg = packageInfoCache.get(pkgToPkgKey(packagePolicy.package))!; + + // Special handling for Universal Profiling packages, as it does not use data streams _only_, + // but also indices that do not adhere to the convention. + if ( + pkg.name === FLEET_UNIVERSAL_PROFILING_SYMBOLIZER_PACKAGE || + pkg.name === FLEET_UNIVERSAL_PROFILING_COLLECTOR_PACKAGE + ) { + return universalProfilingPermissions(packagePolicy.id); + } + + if (pkg.name === FLEET_APM_PACKAGE) { + return apmPermissions(packagePolicy.id); + } + + const dataStreams = getNormalizedDataStreams(pkg); + if (!dataStreams || dataStreams.length === 0) { + return [packagePolicy.name, undefined]; + } + + let dataStreamsForPermissions: DataStreamMeta[]; + + switch (pkg.name) { + case 'endpoint': + // - Endpoint doesn't store the `data_stream` metadata in + // `packagePolicy.inputs`, so we will use _all_ data_streams from the + // package. + dataStreamsForPermissions = dataStreams; + break; + + case 'apm': + // - APM doesn't store the `data_stream` metadata in + // `packagePolicy.inputs`, so we will use _all_ data_streams from + // the package. + dataStreamsForPermissions = dataStreams; + break; + + case 'osquery_manager': + // - Osquery manager doesn't store the `data_stream` metadata in + // `packagePolicy.inputs`, so we will use _all_ data_streams from + // the package. + dataStreamsForPermissions = dataStreams; + break; + + default: + // - Normal packages store some of the `data_stream` metadata in + // `packagePolicy.inputs[].streams[].data_stream` + // - The rest of the metadata needs to be fetched from the + // `data_stream` object in the package. The link is + // `packagePolicy.inputs[].type == dataStreams.streams[].input` + // - Some packages (custom logs) have a compiled dataset, stored in + // `input.streams.compiled_stream.data_stream.dataset` + dataStreamsForPermissions = packagePolicy.inputs + .filter((i) => i.enabled) + .flatMap((input) => { + if (!input.streams) { + return []; + } + + const dataStreams_: DataStreamMeta[] = []; + + input.streams + .filter((s) => s.enabled) + .forEach((stream) => { + if (!('data_stream' in stream)) { + return; + } + + const ds: DataStreamMeta = { + type: stream.data_stream.type, + dataset: + stream.compiled_stream?.data_stream?.dataset ?? stream.data_stream.dataset, + }; + + if (stream.data_stream.elasticsearch) { + ds.elasticsearch = stream.data_stream.elasticsearch; + } + + dataStreams_.push(ds); + }); + + return dataStreams_; + }); + } + + let clusterRoleDescriptor = {}; + const cluster = packagePolicy?.elasticsearch?.privileges?.cluster ?? []; + if (cluster.length > 0) { + clusterRoleDescriptor = { + cluster, + }; } - ); - return Object.fromEntries(await Promise.all(permissionEntries)); + return [ + packagePolicy.id, + { + indices: dataStreamsForPermissions.map((ds) => + getDataStreamPrivileges(ds, packagePolicy.namespace) + ), + ...clusterRoleDescriptor, + }, + ]; + }); + + return Object.fromEntries(permissionEntries); } export interface DataStreamMeta { @@ -171,7 +179,10 @@ export interface DataStreamMeta { }; } -export function getDataStreamPrivileges(dataStream: DataStreamMeta, namespace: string = '*') { +export function getDataStreamPrivileges( + dataStream: DataStreamMeta, + namespace: string = '*' +): SecurityIndicesPrivileges { let index = dataStream.hidden ? `.${dataStream.type}-` : `${dataStream.type}-`; // Determine dataset @@ -200,7 +211,7 @@ export function getDataStreamPrivileges(dataStream: DataStreamMeta, namespace: s }; } -async function universalProfilingPermissions(packagePolicyId: string): Promise<[string, any]> { +function universalProfilingPermissions(packagePolicyId: string): [string, SecurityRoleDescriptor] { const profilingIndexPattern = 'profiling-*'; return [ packagePolicyId, @@ -214,3 +225,22 @@ async function universalProfilingPermissions(packagePolicyId: string): Promise<[ }, ]; } + +function apmPermissions(packagePolicyId: string): [string, SecurityRoleDescriptor] { + return [ + packagePolicyId, + { + cluster: ['cluster:monitor/main'], + indices: [ + { + names: ['traces-*', 'logs-*', 'metrics-*'], + privileges: ['auto_configure', 'create_doc'], + }, + { + names: ['traces-apm.sampled-*'], + privileges: ['auto_configure', 'create_doc', 'maintenance', 'monitor', 'read'], + }, + ], + }, + ]; +} diff --git a/x-pack/test/apm_api_integration/common/bettertest.ts b/x-pack/test/apm_api_integration/common/bettertest.ts index 41580983917a3d..83f7da5725db84 100644 --- a/x-pack/test/apm_api_integration/common/bettertest.ts +++ b/x-pack/test/apm_api_integration/common/bettertest.ts @@ -11,20 +11,32 @@ import request from 'superagent'; type HttpMethod = 'get' | 'post' | 'put' | 'delete'; -export type BetterTest = (options: { +export type BetterTest = (options: BetterTestOptions) => Promise>; + +interface BetterTestOptions { pathname: string; query?: Record; method?: HttpMethod; body?: any; -}) => Promise<{ status: number; body: T }>; +} + +interface BetterTestResponse { + status: number; + body: T; +} /* * This is a wrapper around supertest that throws an error if the response status is not 200. * This is useful for tests that expect a 200 response * It also makes it easier to debug tests that fail because of a 500 response. */ -export function getBettertest(st: supertest.SuperTest): BetterTest { - return async ({ pathname, method = 'get', query, body }) => { +export function getBettertest(st: supertest.SuperTest) { + return async ({ + pathname, + method = 'get', + query, + body, + }: BetterTestOptions): Promise> => { const url = format({ pathname, query }); let res: request.Response; diff --git a/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy.spec.ts b/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy.spec.ts index 0ba4336b36e0c0..d78a6fa6d99d66 100644 --- a/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy.spec.ts +++ b/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy.spec.ts @@ -5,7 +5,6 @@ * 2.0. */ -import * as Url from 'url'; import { PackagePolicy } from '@kbn/fleet-plugin/common'; import { AGENT_CONFIG_PATH, @@ -17,7 +16,7 @@ import expect from '@kbn/expect'; import { get } from 'lodash'; import type { SourceMap } from '@kbn/apm-plugin/server/routes/source_maps/route'; import { APIReturnType } from '@kbn/apm-plugin/public/services/rest/create_call_apm_api'; -import { createEsClientForTesting } from '@kbn/test'; +import { createEsClientForFtrConfig } from '@kbn/test'; import { APM_AGENT_CONFIGURATION_INDEX, APM_SOURCE_MAP_INDEX, @@ -30,7 +29,7 @@ import { deletePackagePolicy, getPackagePolicy, setupFleet, -} from './apm_package_policy_setup'; +} from './helpers'; import { getBettertest } from '../../common/bettertest'; import { expectToReject } from '../../common/utils/expect_to_reject'; @@ -41,14 +40,10 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { const supertest = getService('supertest'); const es = getService('es'); const bettertest = getBettertest(supertest); + const configService = getService('config'); function createEsClientWithApiKeyAuth({ id, apiKey }: { id: string; apiKey: string }) { - const config = getService('config'); - return createEsClientForTesting({ - esUrl: Url.format(config.get('servers.elasticsearch')), - requestTimeout: config.get('timeouts.esRequestTimeout'), - auth: { apiKey: { id, api_key: apiKey } }, - }); + return createEsClientForFtrConfig(configService, { auth: { apiKey: { id, api_key: apiKey } } }); } async function createConfiguration(configuration: any) { @@ -120,8 +115,8 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { before(async () => { await setupFleet(bettertest); - agentPolicyId = await createAgentPolicy(bettertest); - packagePolicyId = await createPackagePolicy(bettertest, agentPolicyId); + agentPolicyId = await createAgentPolicy({ bettertest }); + packagePolicyId = await createPackagePolicy({ bettertest, agentPolicyId }); apmPackagePolicy = await getPackagePolicy(bettertest, packagePolicyId); // make sure to get the latest package policy }); diff --git a/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts b/x-pack/test/apm_api_integration/tests/fleet/helpers.ts similarity index 63% rename from x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts rename to x-pack/test/apm_api_integration/tests/fleet/helpers.ts index aeb2f00fb93608..9cedf76b221b16 100644 --- a/x-pack/test/apm_api_integration/tests/fleet/apm_package_policy_setup.ts +++ b/x-pack/test/apm_api_integration/tests/fleet/helpers.ts @@ -11,15 +11,23 @@ export function setupFleet(bettertest: BetterTest) { return bettertest({ pathname: '/api/fleet/setup', method: 'post' }); } -export async function createAgentPolicy(bettertest: BetterTest, id?: string) { +export async function createAgentPolicy({ + bettertest, + id, + name = 'test_agent_policy', +}: { + bettertest: BetterTest; + id?: string; + name?: string; +}) { const agentPolicyResponse = await bettertest<{ item: AgentPolicy }>({ pathname: '/api/fleet/agent_policies', method: 'post', query: { sys_monitoring: true }, body: { - name: 'test_agent_policy', - description: '', id, + name, + description: '', namespace: 'default', monitoring_enabled: ['logs', 'metrics'], }, @@ -28,11 +36,17 @@ export async function createAgentPolicy(bettertest: BetterTest, id?: string) { return agentPolicyResponse.body.item.id; } -export async function createPackagePolicy( - bettertest: BetterTest, - agentPolicyId: string, - id?: string -) { +export async function createPackagePolicy({ + bettertest, + agentPolicyId, + id, + name = 'apm-integration-test-policy', +}: { + bettertest: BetterTest; + agentPolicyId: string; + id?: string; + name?: string; +}) { // Get version of available APM package const apmPackageResponse = await bettertest<{ item: any }>({ pathname: `/api/fleet/epm/packages/apm`, @@ -44,7 +58,7 @@ export async function createPackagePolicy( pathname: '/api/fleet/package_policies', method: 'post', body: { - name: 'apm-integration-test-policy', + name, description: '', namespace: 'default', policy_id: agentPolicyId, @@ -82,3 +96,36 @@ export async function getPackagePolicy( }); return res.body.item; } + +async function getAgentPolicyByName(bettertest: BetterTest, name: string): Promise { + const res = await bettertest<{ items: PackagePolicy[] }>({ + pathname: `/api/fleet/agent_policies`, + query: { + full: true, + kuery: `name:"${name}"`, + }, + }); + return res.body.items[0]; +} + +export async function deleteAgentPolicyAndPackagePolicyByName({ + bettertest, + agentPolicyName, + packagePolicyName, +}: { + bettertest: BetterTest; + agentPolicyName: string; + packagePolicyName: string; +}) { + const agentPolicy = await getAgentPolicyByName(bettertest, agentPolicyName); + + const agentPolicyId = agentPolicy.id; + // @ts-expect-error + const packagePolicies = agentPolicy.package_policies as PackagePolicy[]; + const packagePolicyId = packagePolicies.find( + (packagePolicy) => packagePolicy.name === packagePolicyName + )!.id; + + await deleteAgentPolicy(bettertest, agentPolicyId); + await deletePackagePolicy(bettertest, packagePolicyId); +} diff --git a/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts b/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts new file mode 100644 index 00000000000000..982d37b8027923 --- /dev/null +++ b/x-pack/test/apm_api_integration/tests/fleet/input_only_package.spec.ts @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { apm, timerange } from '@kbn/apm-synthtrace-client'; +import { ApmSynthtraceEsClient, createLogger, LogLevel } from '@kbn/apm-synthtrace'; +import expect from '@kbn/expect'; +import { createEsClientForFtrConfig } from '@kbn/test'; +import { ApmDocumentType } from '@kbn/apm-plugin/common/document_type'; +import { RollupInterval } from '@kbn/apm-plugin/common/rollup'; +import { SecurityRoleDescriptor } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import pRetry from 'p-retry'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { getBettertest } from '../../common/bettertest'; +import { + createAgentPolicy, + createPackagePolicy, + deleteAgentPolicyAndPackagePolicyByName, + setupFleet, +} from './helpers'; +import { ApmApiClient } from '../../common/config'; + +export default function ApiTest(ftrProviderContext: FtrProviderContext) { + const { getService } = ftrProviderContext; + const registry = getService('registry'); + const apmApiClient = getService('apmApiClient'); + const supertest = getService('supertest'); + const es = getService('es'); + const log = getService('log'); + const bettertest = getBettertest(supertest); + const config = getService('config'); + const synthtraceKibanaClient = getService('synthtraceKibanaClient'); + const synthtraceEsClient = getService('synthtraceEsClient'); + + const API_KEY_NAME = 'apm_api_key_testing'; + const APM_AGENT_POLICY_NAME = 'apm_agent_policy_testing'; + const APM_PACKAGE_POLICY_NAME = 'apm_package_policy_testing'; + + function createEsClientWithApiKey({ id, apiKey }: { id: string; apiKey: string }) { + return createEsClientForFtrConfig(config, { auth: { apiKey: { id, api_key: apiKey } } }); + } + + function createEsClientWithToken(token: string) { + return createEsClientForFtrConfig(config, { auth: { bearer: token } }); + } + + async function getApiKeyForServiceAccount( + serviceAccount: string, + permissions: SecurityRoleDescriptor + ) { + const { token } = await es.security.createServiceToken({ + namespace: 'elastic', + service: serviceAccount, + }); + + const esClientScoped = createEsClientWithToken(token.value); + return esClientScoped.security.createApiKey({ + body: { + name: API_KEY_NAME, + role_descriptors: { + apmFleetPermissions: permissions, + }, + }, + }); + } + + async function getSynthtraceClientWithApiKey({ + id, + api_key: apiKey, + }: { + id: string; + api_key: string; + }) { + const esClient = createEsClientWithApiKey({ id, apiKey }); + const kibanaVersion = await synthtraceKibanaClient.fetchLatestApmPackageVersion(); + return new ApmSynthtraceEsClient({ + client: esClient, + logger: createLogger(LogLevel.info), + version: kibanaVersion, + refreshAfterIndex: true, + }); + } + + registry.when('APM package policy', { config: 'basic', archives: [] }, () => { + async function getAgentPolicyPermissions(agentPolicyId: string, packagePolicyId: string) { + const res = await bettertest<{ + item: { output_permissions: { default: Record } }; + }>({ + pathname: `/api/fleet/agent_policies/${agentPolicyId}/full`, + method: 'get', + }); + + return res.body.item.output_permissions.default[packagePolicyId]; + } + + describe('input only package', () => { + let agentPolicyId: string; + let packagePolicyId: string; + let permissions: SecurityRoleDescriptor; + + async function cleanAll() { + try { + await synthtraceEsClient.clean(); + await es.security.invalidateApiKey({ name: API_KEY_NAME }); + await deleteAgentPolicyAndPackagePolicyByName({ + bettertest, + agentPolicyName: APM_AGENT_POLICY_NAME, + packagePolicyName: APM_PACKAGE_POLICY_NAME, + }); + } catch (e) { + log.info('Nothing to clean'); + } + } + + before(async () => { + await cleanAll(); + + await setupFleet(bettertest); + agentPolicyId = await createAgentPolicy({ bettertest, name: APM_AGENT_POLICY_NAME }); + packagePolicyId = await createPackagePolicy({ + bettertest, + agentPolicyId, + name: APM_PACKAGE_POLICY_NAME, + }); + + permissions = await getAgentPolicyPermissions(agentPolicyId, packagePolicyId); + }); + + after(async () => { + await cleanAll(); + }); + + it('has permissions in the agent policy', async () => { + expect(permissions).to.eql({ + cluster: ['cluster:monitor/main'], + indices: [ + { + names: ['traces-*', 'logs-*', 'metrics-*'], + privileges: ['auto_configure', 'create_doc'], + }, + { + names: ['traces-apm.sampled-*'], + privileges: ['auto_configure', 'create_doc', 'maintenance', 'monitor', 'read'], + }, + ], + }); + }); + + describe('when ingesting events using the scoped api key', () => { + let scenario: ReturnType; + + before(async () => { + scenario = getSynthtraceScenario(); + + // get api key scoped to the specified permissions and created with the fleet-server service account. This ensures that the api key is not created with more permissions than fleet-server is able to. + const apiKey = await getApiKeyForServiceAccount('fleet-server', permissions); + + // create a synthtrace client scoped to the api key. This verifies that the api key has permissions to write to the APM indices. + const scopedSynthtraceEsClient = await getSynthtraceClientWithApiKey(apiKey); + await scopedSynthtraceEsClient.index(scenario.events); + }); + + it('the events can be seen on the Service Inventory Page', async () => { + const apmServices = await getApmServices(apmApiClient, scenario.start, scenario.end); + expect(apmServices[0].serviceName).to.be('opbeans-java'); + expect(apmServices[0].environments?.[0]).to.be('ingested-via-fleet'); + expect(apmServices[0].latency).to.be(2550000); + expect(apmServices[0].throughput).to.be(2); + expect(apmServices[0].transactionErrorRate).to.be(0.5); + }); + }); + }); + }); +} + +function getApmServices(apmApiClient: ApmApiClient, start: string, end: string) { + return pRetry(async () => { + const res = await apmApiClient.readUser({ + endpoint: 'GET /internal/apm/services', + params: { + query: { + start, + end, + probability: 1, + environment: 'ENVIRONMENT_ALL', + kuery: '', + documentType: ApmDocumentType.TransactionMetric, + rollupInterval: RollupInterval.OneMinute, + }, + }, + }); + + if (res.body.items.length === 0 || !res.body.items[0].latency) { + throw new Error(`Timed-out: No APM Services were found`); + } + + return res.body.items; + }); +} + +function getSynthtraceScenario() { + const start = new Date('2023-09-01T00:00:00.000Z').getTime(); + const end = new Date('2023-09-01T00:02:00.000Z').getTime(); + + const opbeansJava = apm + .service({ name: 'opbeans-java', environment: 'ingested-via-fleet', agentName: 'java' }) + .instance('instance'); + + const events = timerange(start, end) + .ratePerMinute(1) + .generator((timestamp) => { + return [ + opbeansJava + .transaction({ transactionName: 'tx-java' }) + .timestamp(timestamp) + .duration(5000) + .success(), + + opbeansJava + .transaction({ transactionName: 'tx-java' }) + .timestamp(timestamp) + .duration(100) + .failure() + .errors(opbeansJava.error({ message: 'some error' }).timestamp(timestamp + 50)), + ]; + }); + + return { + start: new Date(start).toISOString(), + end: new Date(end).toISOString(), + events, + }; +} diff --git a/x-pack/test/apm_api_integration/tests/fleet/migration_check.spec.ts b/x-pack/test/apm_api_integration/tests/fleet/migration_check.spec.ts index 1f89f65c536b9a..74cf37c957d135 100644 --- a/x-pack/test/apm_api_integration/tests/fleet/migration_check.spec.ts +++ b/x-pack/test/apm_api_integration/tests/fleet/migration_check.spec.ts @@ -13,7 +13,7 @@ import { deleteAgentPolicy, deletePackagePolicy, setupFleet, -} from './apm_package_policy_setup'; +} from './helpers'; import { getBettertest } from '../../common/bettertest'; export default function ApiTest(ftrProviderContext: FtrProviderContext) { @@ -76,7 +76,7 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { }); describe('with Cloud agent policy', () => { before(async () => { - await createAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud'); + await createAgentPolicy({ bettertest, id: 'policy-elastic-agent-on-cloud' }); }); after(async () => { await deleteAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud'); @@ -92,7 +92,7 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { describe('has_cloud_apm_package_policy', () => { before(async () => { - await createAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud'); + await createAgentPolicy({ bettertest, id: 'policy-elastic-agent-on-cloud' }); }); after(async () => { await deleteAgentPolicy(bettertest, 'policy-elastic-agent-on-cloud'); @@ -107,7 +107,11 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { }); describe('with Cloud APM package policy', () => { before(async () => { - await createPackagePolicy(bettertest, 'policy-elastic-agent-on-cloud', 'apm'); + await createPackagePolicy({ + bettertest, + agentPolicyId: 'policy-elastic-agent-on-cloud', + id: 'apm', + }); }); after(async () => { await deletePackagePolicy(bettertest, 'apm'); @@ -125,7 +129,7 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { describe('has_apm_integrations', () => { before(async () => { - await createAgentPolicy(bettertest, 'test-agent-policy'); + await createAgentPolicy({ bettertest, id: 'test-agent-policy' }); }); after(async () => { await deleteAgentPolicy(bettertest, 'test-agent-policy'); @@ -139,7 +143,11 @@ export default function ApiTest(ftrProviderContext: FtrProviderContext) { }); describe('with custom APM package policy', () => { before(async () => { - await createPackagePolicy(bettertest, 'test-agent-policy', 'test-apm-package-policy'); + await createPackagePolicy({ + bettertest, + agentPolicyId: 'test-agent-policy', + id: 'test-apm-package-policy', + }); }); after(async () => { await deletePackagePolicy(bettertest, 'test-apm-package-policy');