diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index 7c52908cc4e437..18102c1224431e 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -588,7 +588,7 @@ activities. |{kib-repo}blob/{branch}/x-pack/plugins/fields_metadata/README.md[fieldsMetadata] -|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. +|The @kbn/fields-metadata-plugin is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS/Metadata definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. |{kib-repo}blob/{branch}/x-pack/plugins/file_upload[fileUpload] diff --git a/x-pack/plugins/fields_metadata/README.md b/x-pack/plugins/fields_metadata/README.md index d08b18a0293454..ea561c52febcef 100755 --- a/x-pack/plugins/fields_metadata/README.md +++ b/x-pack/plugins/fields_metadata/README.md @@ -1,6 +1,6 @@ # Fields Metadata Plugin -The `@kbn/fields-metadata-plugin` is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. +The `@kbn/fields-metadata-plugin` is designed to provide a centralized and asynchronous way to consume field metadata across Kibana. This plugin addresses the need for on-demand retrieval of field metadata from static ECS/Metadata definitions and integration manifests, with the flexibility to extend to additional resolution sources in the future. ## Components and Mechanisms diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts index 0ddda927de676f..dffc3fd1217b12 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/models/fields_metadata_dictionary.ts @@ -14,6 +14,10 @@ export type FieldsMetadataMap = Record; export class FieldsMetadataDictionary { private constructor(private readonly fields: FieldsMetadataMap) {} + getFields() { + return this.fields; + } + pick(attributes: FieldAttribute[]): Record { return mapValues(this.fields, (field) => field.pick(attributes)); } diff --git a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts index 8a1327e363aadc..a2975bffee46a4 100644 --- a/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts +++ b/x-pack/plugins/fields_metadata/common/fields_metadata/types.ts @@ -7,10 +7,12 @@ import { EcsFlat } from '@elastic/ecs'; import * as rt from 'io-ts'; +import { MetadataFields } from '../metadata_fields'; export const fieldSourceRT = rt.keyof({ ecs: null, integration: null, + metadata: null, unknown: null, }); @@ -63,6 +65,7 @@ const optionalMetadataPlainRT = rt.partial({ short: rt.string, source: fieldSourceRT, type: rt.string, + documentation_url: rt.string, }); export const partialFieldMetadataPlainRT = rt.intersection([ @@ -80,11 +83,14 @@ export const fieldAttributeRT = rt.union([ rt.keyof(optionalMetadataPlainRT.props), ]); +export type AnyFieldName = string & {}; +export type TMetadataFields = typeof MetadataFields; +export type MetadataFieldName = keyof TMetadataFields; export type TEcsFields = typeof EcsFlat; export type EcsFieldName = keyof TEcsFields; -export type IntegrationFieldName = string; +export type IntegrationFieldName = AnyFieldName; -export type FieldName = EcsFieldName | (IntegrationFieldName & {}); +export type FieldName = MetadataFieldName | EcsFieldName | IntegrationFieldName; export type FieldMetadataPlain = rt.TypeOf; export type PartialFieldMetadataPlain = rt.TypeOf; diff --git a/x-pack/plugins/fields_metadata/common/index.ts b/x-pack/plugins/fields_metadata/common/index.ts index 8daf749f74261e..5417ac4de212fd 100644 --- a/x-pack/plugins/fields_metadata/common/index.ts +++ b/x-pack/plugins/fields_metadata/common/index.ts @@ -7,6 +7,7 @@ export { fieldMetadataPlainRT } from './fields_metadata/types'; export type { + AnyFieldName, EcsFieldName, FieldAttribute, FieldMetadataPlain, @@ -14,6 +15,7 @@ export type { IntegrationFieldName, PartialFieldMetadataPlain, TEcsFields, + TMetadataFields, } from './fields_metadata/types'; export { FieldMetadata } from './fields_metadata/models/field_metadata'; diff --git a/x-pack/plugins/fields_metadata/common/metadata_fields.ts b/x-pack/plugins/fields_metadata/common/metadata_fields.ts new file mode 100644 index 00000000000000..4def656fc438c4 --- /dev/null +++ b/x-pack/plugins/fields_metadata/common/metadata_fields.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const MetadataFields = { + _index: { + dashed_name: 'index', + description: + 'The index to which the document belongs. This metadata field specifies the exact index name in which the document is stored.', + example: 'index_1', + flat_name: '_index', + name: '_index', + short: 'The index to which the document belongs.', + type: 'keyword', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html', + }, + _id: { + dashed_name: 'id', + description: + 'The document’s ID. This unique identifier is used to fetch, update, or delete a document within an index.', + example: '1', + flat_name: '_id', + name: '_id', + short: 'The document’s ID.', + type: 'keyword', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-id-field.html', + }, + _source: { + dashed_name: 'source', + description: + 'The original JSON representing the body of the document. This field contains all the source data that was provided at the time of indexing.', + example: '{"user": "John Doe", "message": "Hello"}', + flat_name: '_source', + name: '_source', + short: 'The original JSON representing the body of the document.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html', + }, + _size: { + dashed_name: 'size', + description: + 'The size of the _source field in bytes. Provided by the mapper-size plugin, this metadata field helps in understanding the storage impact of the document.', + example: '150', + flat_name: '_size', + name: '_size', + short: 'The size of the _source field in bytes, provided by the mapper-size plugin.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/plugins/current/mapper-size.html', + }, + _doc_count: { + dashed_name: 'doc_count', + description: + 'A custom field used for storing document counts when a document represents pre-aggregated data. It helps in scenarios involving pre-computed data aggregation.', + example: '42', + flat_name: '_doc_count', + name: '_doc_count', + short: + 'A custom field used for storing doc counts when a document represents pre-aggregated data.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-doc-count-field.html', + }, + _field_names: { + dashed_name: 'field_names', + description: + 'All fields in the document which contain non-null values. This metadata field lists the field names that have valid data.', + example: '["user", "message"]', + flat_name: '_field_names', + name: '_field_names', + short: 'Fields with non-null values.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-field-names-field.html', + }, + _ignored: { + dashed_name: 'ignored', + description: + 'All fields in the document that have been ignored at index time because of ignore_malformed. It indicates fields that were not indexed due to malformation.', + example: '["malformed_field"]', + flat_name: '_ignored', + name: '_ignored', + short: 'Fields ignored during indexing.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-ignored-field.html', + }, + _routing: { + dashed_name: 'routing', + description: + 'A custom routing value which routes a document to a particular shard. This field is used to control the shard placement of a document.', + example: 'user_routing_value', + flat_name: '_routing', + name: '_routing', + short: 'Custom shard routing value.', + type: 'keyword', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-routing-field.html', + }, + _meta: { + dashed_name: 'meta', + description: + 'Application specific metadata. This field can store any custom metadata relevant to the application using Elasticsearch.', + example: '{"app": "my_app"}', + flat_name: '_meta', + name: '_meta', + short: 'Custom application metadata.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-meta-field.html', + }, + _tier: { + dashed_name: 'tier', + description: + 'The current data tier preference of the index to which the document belongs. It helps in managing the index’s storage tier.', + example: 'hot', + flat_name: '_tier', + name: '_tier', + short: 'Index data tier preference.', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-tier-field.html', + }, +}; diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts index 0d35dc70fd6782..e693725a93d89c 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.test.ts @@ -4,11 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { FieldMetadata, TEcsFields } from '../../../common'; +import { FieldMetadata, TEcsFields, TMetadataFields } from '../../../common'; import { loggerMock } from '@kbn/logging-mocks'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; +import { MetadataFieldsRepository } from './repositories/metadata_fields_repository'; const ecsFields = { '@timestamp': { @@ -26,6 +27,21 @@ const ecsFields = { }, } as TEcsFields; +const metadataFields = { + _index: { + dashed_name: 'index', + description: + 'The index to which the document belongs. This metadata field specifies the exact index name in which the document is stored.', + example: 'index_1', + flat_name: '_index', + name: '_index', + short: 'The index to which the document belongs.', + type: 'keyword', + documentation_url: + 'https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-index-field.html', + }, +} as TMetadataFields; + const integrationFields = { '1password.item_usages': { 'onepassword.client.platform_version': { @@ -46,6 +62,7 @@ const integrationFields = { describe('FieldsMetadataClient class', () => { const logger = loggerMock.create(); const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); + const metadataFieldsRepository = MetadataFieldsRepository.create({ metadataFields }); const integrationFieldsExtractor = jest.fn(); integrationFieldsExtractor.mockImplementation(() => Promise.resolve(integrationFields)); @@ -60,12 +77,13 @@ describe('FieldsMetadataClient class', () => { fieldsMetadataClient = FieldsMetadataClient.create({ ecsFieldsRepository, integrationFieldsRepository, + metadataFieldsRepository, logger, }); }); describe('#getByName', () => { - it('should resolve a single ECS FieldMetadata instance by default', async () => { + it('should resolve a single ECS/Metadata FieldMetadata instance by default', async () => { const timestampFieldInstance = await fieldsMetadataClient.getByName('@timestamp'); expect(integrationFieldsExtractor).not.toHaveBeenCalled(); @@ -87,7 +105,7 @@ describe('FieldsMetadataClient class', () => { expect(timestampField.hasOwnProperty('type')).toBeTruthy(); }); - it('should attempt resolving the field from an integration if it does not exist in ECS and the integration and dataset params are provided', async () => { + it('should attempt resolving the field from an integration if it does not exist in ECS/Metadata and the integration and dataset params are provided', async () => { const onePasswordFieldInstance = await fieldsMetadataClient.getByName( 'onepassword.client.platform_version', { integration: '1password', dataset: '1password.item_usages' } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts index e152c143479279..87c9b6547f4f58 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_client.ts @@ -9,12 +9,14 @@ import { Logger } from '@kbn/core/server'; import { FieldName, FieldMetadata, FieldsMetadataDictionary } from '../../../common'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; +import { MetadataFieldsRepository } from './repositories/metadata_fields_repository'; import { IntegrationFieldsSearchParams } from './repositories/types'; import { FindFieldsMetadataOptions, IFieldsMetadataClient } from './types'; interface FieldsMetadataClientDeps { logger: Logger; ecsFieldsRepository: EcsFieldsRepository; + metadataFieldsRepository: MetadataFieldsRepository; integrationFieldsRepository: IntegrationFieldsRepository; } @@ -22,6 +24,7 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { private constructor( private readonly logger: Logger, private readonly ecsFieldsRepository: EcsFieldsRepository, + private readonly metadataFieldsRepository: MetadataFieldsRepository, private readonly integrationFieldsRepository: IntegrationFieldsRepository ) {} @@ -31,8 +34,13 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { ): Promise { this.logger.debug(`Retrieving field metadata for: ${fieldName}`); - // 1. Try resolving from ecs static metadata - let field = this.ecsFieldsRepository.getByName(fieldName); + // 1. Try resolving from metadata-fields static metadata + let field = this.metadataFieldsRepository.getByName(fieldName); + + // 2. Try resolving from ecs static metadata + if (!field) { + field = this.ecsFieldsRepository.getByName(fieldName); + } // 2. Try searching for the fiels in the Elastic Package Registry if (!field && integration) { @@ -48,7 +56,10 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { dataset, }: FindFieldsMetadataOptions = {}): Promise { if (!fieldNames) { - return this.ecsFieldsRepository.find(); + return FieldsMetadataDictionary.create({ + ...this.metadataFieldsRepository.find().getFields(), + ...this.ecsFieldsRepository.find().getFields(), + }); } const fields: Record = {}; @@ -66,8 +77,14 @@ export class FieldsMetadataClient implements IFieldsMetadataClient { public static create({ logger, ecsFieldsRepository, + metadataFieldsRepository, integrationFieldsRepository, }: FieldsMetadataClientDeps) { - return new FieldsMetadataClient(logger, ecsFieldsRepository, integrationFieldsRepository); + return new FieldsMetadataClient( + logger, + ecsFieldsRepository, + metadataFieldsRepository, + integrationFieldsRepository + ); } } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts index 391da465e9a1fe..8313f0337d7695 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/fields_metadata_service.ts @@ -10,8 +10,10 @@ import { Logger } from '@kbn/core/server'; import { FieldsMetadataClient } from './fields_metadata_client'; import { EcsFieldsRepository } from './repositories/ecs_fields_repository'; import { IntegrationFieldsRepository } from './repositories/integration_fields_repository'; +import { MetadataFieldsRepository } from './repositories/metadata_fields_repository'; import { IntegrationFieldsExtractor } from './repositories/types'; import { FieldsMetadataServiceSetup, FieldsMetadataServiceStart } from './types'; +import { MetadataFields as metadataFields } from '../../../common/metadata_fields'; export class FieldsMetadataService { private integrationFieldsExtractor: IntegrationFieldsExtractor = () => Promise.resolve({}); @@ -30,6 +32,7 @@ export class FieldsMetadataService { const { logger, integrationFieldsExtractor } = this; const ecsFieldsRepository = EcsFieldsRepository.create({ ecsFields }); + const metadataFieldsRepository = MetadataFieldsRepository.create({ metadataFields }); const integrationFieldsRepository = IntegrationFieldsRepository.create({ integrationFieldsExtractor, }); @@ -39,6 +42,7 @@ export class FieldsMetadataService { return FieldsMetadataClient.create({ logger, ecsFieldsRepository, + metadataFieldsRepository, integrationFieldsRepository, }); }, diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts index 05f34bbf0ee600..d8fb947bdd6c5b 100644 --- a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/ecs_fields_repository.ts @@ -7,18 +7,18 @@ import mapValues from 'lodash/mapValues'; import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary'; -import { FieldMetadata, FieldName, TEcsFields } from '../../../../common'; +import { AnyFieldName, EcsFieldName, FieldMetadata, TEcsFields } from '../../../../common'; interface EcsFieldsRepositoryDeps { ecsFields: TEcsFields; } interface FindOptions { - fieldNames?: FieldName[]; + fieldNames?: EcsFieldName[]; } export class EcsFieldsRepository { - private readonly ecsFields: Record; + private readonly ecsFields: Record; private constructor(ecsFields: TEcsFields) { this.ecsFields = mapValues(ecsFields, (field) => @@ -26,8 +26,8 @@ export class EcsFieldsRepository { ); } - getByName(fieldName: FieldName): FieldMetadata | undefined { - return this.ecsFields[fieldName]; + getByName(fieldName: EcsFieldName | AnyFieldName): FieldMetadata | undefined { + return this.ecsFields[fieldName as EcsFieldName]; } find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary { @@ -43,7 +43,7 @@ export class EcsFieldsRepository { } return fieldsMetadata; - }, {} as Record); + }, {} as Record); return FieldsMetadataDictionary.create(fields); } diff --git a/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/metadata_fields_repository.ts b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/metadata_fields_repository.ts new file mode 100644 index 00000000000000..1276c672acb629 --- /dev/null +++ b/x-pack/plugins/fields_metadata/server/services/fields_metadata/repositories/metadata_fields_repository.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import mapValues from 'lodash/mapValues'; +import { MetadataFieldName } from '../../../../common/fields_metadata'; +import { FieldsMetadataDictionary } from '../../../../common/fields_metadata/models/fields_metadata_dictionary'; +import { AnyFieldName, FieldMetadata, TMetadataFields } from '../../../../common'; + +interface MetadataFieldsRepositoryDeps { + metadataFields: TMetadataFields; +} + +interface FindOptions { + fieldNames?: MetadataFieldName[]; +} + +export class MetadataFieldsRepository { + private readonly metadataFields: Record; + + private constructor(metadataFields: TMetadataFields) { + this.metadataFields = mapValues(metadataFields, (field) => + FieldMetadata.create({ ...field, source: 'metadata' }) + ); + } + + getByName(fieldName: MetadataFieldName | AnyFieldName): FieldMetadata | undefined { + return this.metadataFields[fieldName as MetadataFieldName]; + } + + find({ fieldNames }: FindOptions = {}): FieldsMetadataDictionary { + if (!fieldNames) { + return FieldsMetadataDictionary.create(this.metadataFields); + } + + const fields = fieldNames.reduce((fieldsMetadata, fieldName) => { + const field = this.getByName(fieldName); + + if (field) { + fieldsMetadata[fieldName] = field; + } + + return fieldsMetadata; + }, {} as Record); + + return FieldsMetadataDictionary.create(fields); + } + + public static create({ metadataFields }: MetadataFieldsRepositoryDeps) { + return new MetadataFieldsRepository(metadataFields); + } +}